diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 0000000000..355fb965ca --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,3 @@ +# Whole-repo Prettier reformat (CLDSRV-1002), no behaviour change. +# Run once locally: git config blame.ignoreRevsFile .git-blame-ignore-revs +02f94246fe9d3f9e4484d526adc81a73e90a0540 diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 6553e014e9..636b0e4395 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -9,7 +9,7 @@ the **[Zenko Forum](http://forum.zenko.io/)**. > Questions opened as GitHub issues will systematically be closed, and moved to > the [Zenko Forum](http://forum.zenko.io/). --------------------------------------------------------------------------------- +--- ## Avoiding duplicates @@ -21,7 +21,7 @@ any duplicates already open: - if there is a duplicate, please do not open your issue, and add a comment to the existing issue instead. --------------------------------------------------------------------------------- +--- ## Bug report information @@ -52,7 +52,7 @@ Describe the results you expected - distribution/OS, - optional: anything else you deem helpful to us. --------------------------------------------------------------------------------- +--- ## Feature Request @@ -78,10 +78,10 @@ Please provide use cases for changing the current behavior ### Additional information - Is this request for your company? Y/N - - If Y: Company name: - - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N + - If Y: Company name: + - Are you using any Scality Enterprise Edition products (RING, Zenko EE)? Y/N - Are you willing to contribute this feature yourself? - Position/Title: - How did you hear about us? --------------------------------------------------------------------------------- +--- diff --git a/.github/actions/cleanup-and-coverage/action.yaml b/.github/actions/cleanup-and-coverage/action.yaml index f783b9518d..562d58e1d3 100644 --- a/.github/actions/cleanup-and-coverage/action.yaml +++ b/.github/actions/cleanup-and-coverage/action.yaml @@ -34,7 +34,7 @@ runs: env: INPUT_PROFILES: ${{ inputs.profiles }} working-directory: .github/docker - + - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v5 with: diff --git a/.github/codeql/qlpack.yml b/.github/codeql/qlpack.yml index 8a73ff8cbc..6d7a58f725 100644 --- a/.github/codeql/qlpack.yml +++ b/.github/codeql/qlpack.yml @@ -1,4 +1,4 @@ name: scality/cloudserver-async-migration version: 0.0.1 dependencies: - codeql/javascript-all: "*" + codeql/javascript-all: '*' diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 656e5f645f..7c4b762e2a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -6,7 +6,7 @@ updates: interval: weekly open-pull-requests-limit: 5 ignore: - - dependency-name: "scality/vault" - update-types: ["version-update:semver-major"] - - dependency-name: "scality/metadata" - update-types: ["version-update:semver-major"] + - dependency-name: 'scality/vault' + update-types: ['version-update:semver-major'] + - dependency-name: 'scality/metadata' + update-types: ['version-update:semver-major'] diff --git a/.github/docker/config.s3c.json b/.github/docker/config.s3c.json index f0f0744b8d..f2795a325a 100644 --- a/.github/docker/config.s3c.json +++ b/.github/docker/config.s3c.json @@ -35,25 +35,17 @@ "dumpLevel": "error" }, "replicationEndpoints": [ - {"site": "zenko", "servers": ["127.0.0.1:9080"], "echo": false, "default": true}, - {"site": "us-east-2", "servers": ["127.0.0.1:9080"]} + { "site": "zenko", "servers": ["127.0.0.1:9080"], "echo": false, "default": true }, + { "site": "us-east-2", "servers": ["127.0.0.1:9080"] } ], "requests": { "extractClientIPFromHeader": "x-forwarded-for", "extractProtocolFromHeader": "x-forwarded-proto", - "trustedProxyCIDRs": [ - "127.0.0.1/32", - "::ffff:127.0.0.1/128", - "127.0.0.1" - ], + "trustedProxyCIDRs": ["127.0.0.1/32", "::ffff:127.0.0.1/128", "127.0.0.1"], "viaProxy": true }, "multiObjectDeleteEnableOptimizations": false, - "supportedLifecycleRules": [ - "Expiration", - "NoncurrentVersionExpiration", - "AbortIncompleteMultipartUpload" - ], + "supportedLifecycleRules": ["Expiration", "NoncurrentVersionExpiration", "AbortIncompleteMultipartUpload"], "bucketNotificationDestinations": [ { "resource": "target1", diff --git a/.github/docker/docker-compose.sse.yaml b/.github/docker/docker-compose.sse.yaml index f902ba81d8..4b50563154 100644 --- a/.github/docker/docker-compose.sse.yaml +++ b/.github/docker/docker-compose.sse.yaml @@ -5,7 +5,7 @@ services: # root because S3C images needs ownership permission on files and mounted paths user: root command: sh -c "chmod 400 tests/utils/keyfile && yarn start > /artifacts/vault.log 2> /artifacts/vault-stderr.log" - network_mode: "host" + network_mode: 'host' volumes: - /tmp/artifacts/${JOB_NAME}:/artifacts - ./vault-config.json:/conf/config.json:ro diff --git a/.github/docker/md-config.json b/.github/docker/md-config.json index 405c350db5..05afaf5da4 100644 --- a/.github/docker/md-config.json +++ b/.github/docker/md-config.json @@ -11,7 +11,7 @@ "logLevel": "info", "env": { "METADATA_NEW_BUCKETS_VFORMAT": "v0", - "S3_VERSION_ID_ENCODING_TYPE":"hex" + "S3_VERSION_ID_ENCODING_TYPE": "hex" }, "migration": { "deploy": false, diff --git a/.github/docker/vault-config.json b/.github/docker/vault-config.json index 6f558cf5c4..ef00a63126 100644 --- a/.github/docker/vault-config.json +++ b/.github/docker/vault-config.json @@ -68,7 +68,7 @@ "kmsAWS": { "noAwsArn": true, "providerName": "local", - "region": "us-east-1", + "region": "us-east-1", "endpoint": "http://0:8080", "ak": "456", "sk": "123" diff --git a/.github/scripts/check-diff-async.mjs b/.github/scripts/check-diff-async.mjs index 5d4564d193..d2c7f7302f 100644 --- a/.github/scripts/check-diff-async.mjs +++ b/.github/scripts/check-diff-async.mjs @@ -11,17 +11,10 @@ import { Project, SyntaxKind } from 'ts-morph'; const CALLBACK_PARAM_PATTERN = /^(cb|callback|next|done)$/i; function getChangedJsFiles() { - const base = process.env.GITHUB_BASE_REF - ? `origin/${process.env.GITHUB_BASE_REF}` - : 'HEAD'; - const output = execFileSync('git', [ - 'diff', - '--name-only', - '--diff-filter=ACMR', - base, - '--', - '**/*.js', - ], { encoding: 'utf8' }).trim(); + const base = process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : 'HEAD'; + const output = execFileSync('git', ['diff', '--name-only', '--diff-filter=ACMR', base, '--', '**/*.js'], { + encoding: 'utf8', + }).trim(); return output ? output.split('\n').filter(f => f.endsWith('.js')) : []; } @@ -30,9 +23,7 @@ function getChangedJsFiles() { * Get added line numbers for a file in the current diff. */ function getAddedLineNumbers(filePath) { - const base = process.env.GITHUB_BASE_REF - ? `origin/${process.env.GITHUB_BASE_REF}` - : 'HEAD'; + const base = process.env.GITHUB_BASE_REF ? `origin/${process.env.GITHUB_BASE_REF}` : 'HEAD'; const diff = execFileSync('git', ['diff', base, '--', filePath], { encoding: 'utf8' }); const addedLines = new Set(); let currentLine = 0; @@ -69,14 +60,11 @@ const project = new Project({ skipAddingFilesFromTsConfig: true, }); -const filesToCheck = changedFiles.filter(f => - !f.startsWith('tests/') && - !f.startsWith('node_modules/') && - ( - f.startsWith('lib/') || - f.startsWith('bin/') || - !f.includes('/') - ) +const filesToCheck = changedFiles.filter( + f => + !f.startsWith('tests/') && + !f.startsWith('node_modules/') && + (f.startsWith('lib/') || f.startsWith('bin/') || !f.includes('/')), ); if (filesToCheck.length === 0) { console.log('No source JS files in diff (tests and node_modules excluded).'); diff --git a/.github/scripts/cleanupOldGCPBuckets.js b/.github/scripts/cleanupOldGCPBuckets.js index 146fc7bf2a..767a79524f 100644 --- a/.github/scripts/cleanupOldGCPBuckets.js +++ b/.github/scripts/cleanupOldGCPBuckets.js @@ -21,8 +21,7 @@ function buildClient() { if (!accessKeyId || !secretAccessKey) { console.error( - 'Missing required environment variables: ' + - 'AWS_GCP_BACKEND_ACCESS_KEY and AWS_GCP_BACKEND_SECRET_KEY' + 'Missing required environment variables: ' + 'AWS_GCP_BACKEND_ACCESS_KEY and AWS_GCP_BACKEND_SECRET_KEY', ); process.exit(1); } @@ -56,21 +55,23 @@ async function abortMultipartUploads(client, bucketName) { let keyMarker; do { - const res = await client.send(new ListMultipartUploadsCommand({ - Bucket: bucketName, - UploadIdMarker: uploadIdMarker, - KeyMarker: keyMarker, - })); + const res = await client.send( + new ListMultipartUploadsCommand({ + Bucket: bucketName, + UploadIdMarker: uploadIdMarker, + KeyMarker: keyMarker, + }), + ); for (const upload of res.Uploads || []) { - console.log( - ` Aborting MPU: ${upload.Key} (${upload.UploadId})` + console.log(` Aborting MPU: ${upload.Key} (${upload.UploadId})`); + await client.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: upload.Key, + UploadId: upload.UploadId, + }), ); - await client.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: upload.Key, - UploadId: upload.UploadId, - })); } uploadIdMarker = res.NextUploadIdMarker; @@ -82,19 +83,23 @@ async function deleteAllObjects(client, bucketName) { let continuationToken; do { - const res = await client.send(new ListObjectsV2Command({ - Bucket: bucketName, - ContinuationToken: continuationToken, - })); + const res = await client.send( + new ListObjectsV2Command({ + Bucket: bucketName, + ContinuationToken: continuationToken, + }), + ); const objects = res.Contents || []; if (objects.length > 0) { console.log(` Deleting ${objects.length} object(s)...`); for (const obj of objects) { - await client.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.Key, - })); + await client.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.Key, + }), + ); } } @@ -121,9 +126,8 @@ async function main() { const { Buckets = [] } = await client.send(new ListBucketsCommand({})); const now = Date.now(); - const stale = Buckets.filter(b => - b.Name.startsWith(BUCKET_PREFIX) && - now - new Date(b.CreationDate).getTime() > ONE_WEEK_MS + const stale = Buckets.filter( + b => b.Name.startsWith(BUCKET_PREFIX) && now - new Date(b.CreationDate).getTime() > ONE_WEEK_MS, ); if (stale.length === 0) { @@ -131,10 +135,7 @@ async function main() { return; } - console.log( - `Found ${stale.length} stale GCP CI bucket(s) to clean up: ${ - stale.map(b => b.Name).join(', ')}` - ); + console.log(`Found ${stale.length} stale GCP CI bucket(s) to clean up: ${stale.map(b => b.Name).join(', ')}`); for (const bucket of stale) { await cleanupBucket(client, bucket.Name); diff --git a/.github/scripts/count-async-functions.mjs b/.github/scripts/count-async-functions.mjs index 48fd4b65ea..895fb7751c 100644 --- a/.github/scripts/count-async-functions.mjs +++ b/.github/scripts/count-async-functions.mjs @@ -67,13 +67,12 @@ for (const sourceFile of project.getSourceFiles()) { } } -const asyncFunctionPercent = totalFunctions > 0 - ? ((asyncFunctions / totalFunctions) * 100).toFixed(1) - : '0.0'; +const asyncFunctionPercent = totalFunctions > 0 ? ((asyncFunctions / totalFunctions) * 100).toFixed(1) : '0.0'; -const migrationPercent = (asyncFunctions + callbackFunctions) > 0 - ? ((asyncFunctions / (asyncFunctions + callbackFunctions)) * 100).toFixed(1) - : '0.0'; +const migrationPercent = + asyncFunctions + callbackFunctions > 0 + ? ((asyncFunctions / (asyncFunctions + callbackFunctions)) * 100).toFixed(1) + : '0.0'; console.log('=== Async/Await Migration Progress ==='); console.log(`Total functions: ${totalFunctions}`); @@ -84,18 +83,21 @@ console.log(''); console.log(`Migration (trend): ${asyncFunctions}/${asyncFunctions + callbackFunctions} (${migrationPercent}%)`); if (process.env.GITHUB_STEP_SUMMARY) { - appendFileSync(process.env.GITHUB_STEP_SUMMARY, [ - '## Async/Await Migration Progress', - '', - `| Metric | Count |`, - `|--------|-------|`, - `| Total functions | ${totalFunctions} |`, - `| Async functions | ${asyncFunctions} (${asyncFunctionPercent}%) |`, - `| Callback-style functions | ${callbackFunctions} |`, - `| Remaining \`.then()\` chains | ${thenChains} |`, - `| Migration trend (async / (async + callback)) | ${asyncFunctions}/${asyncFunctions + callbackFunctions} (${migrationPercent}%) |`, - '', - ].join('\n')); + appendFileSync( + process.env.GITHUB_STEP_SUMMARY, + [ + '## Async/Await Migration Progress', + '', + `| Metric | Count |`, + `|--------|-------|`, + `| Total functions | ${totalFunctions} |`, + `| Async functions | ${asyncFunctions} (${asyncFunctionPercent}%) |`, + `| Callback-style functions | ${callbackFunctions} |`, + `| Remaining \`.then()\` chains | ${thenChains} |`, + `| Migration trend (async / (async + callback)) | ${asyncFunctions}/${asyncFunctions + callbackFunctions} (${migrationPercent}%) |`, + '', + ].join('\n'), + ); // Output benchmark JSON for visualization const benchmarkData = [ @@ -113,7 +115,7 @@ if (process.env.GITHUB_STEP_SUMMARY) { name: 'Total callback functions', unit: 'count', value: callbackFunctions, - } + }, ]; writeFileSync('async-migration-benchmark.json', JSON.stringify(benchmarkData, null, 2)); } diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 5713ac437e..71bdf56030 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -14,7 +14,7 @@ jobs: tests: - name: 1 minute interval tests file: monitoring/alerts.test.yaml - + - name: 10 seconds interval tests file: monitoring/alerts.10s.test.yaml diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 4476a31afb..56337747ee 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -15,18 +15,11 @@ jobs: steps: - name: Checkout uses: actions/checkout@v6 - with: - fetch-depth: 0 - uses: actions/setup-node@v6 with: node-version: '22.23.1' cache: yarn - name: install dependencies run: yarn install --frozen-lockfile --network-concurrency 1 - - name: Prettier (changed files) - shell: bash - env: - BASE_REF: ${{ github.event.pull_request.base.ref }} - run: | - MERGE_BASE=$(git merge-base HEAD "origin/${BASE_REF}") - yarn run --silent prettier:diff --check "${MERGE_BASE}..HEAD" + - name: Prettier + run: yarn run --silent prettier:check diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c81a539afe..ad63cecfb0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -69,8 +69,7 @@ jobs: push: true context: images/federation provenance: false - build-args: - CLOUDSERVER_VERSION=${{ github.event.inputs.tag }} + build-args: CLOUDSERVER_VERSION=${{ github.event.inputs.tag }} tags: | ghcr.io/${{ github.repository }}:${{ github.event.inputs.tag }}-federation labels: | diff --git a/.nycrc b/.nycrc index 84e748de7c..bdd7c0a39f 100644 --- a/.nycrc +++ b/.nycrc @@ -1,10 +1,5 @@ { "all": true, - "include": [ - "bin/**/*.js", - "lib/**/*.js" - ], - "exclude": [ - "tests/**/*.js" - ] -} \ No newline at end of file + "include": ["bin/**/*.js", "lib/**/*.js"], + "exclude": ["tests/**/*.js"] +} diff --git a/.prettierignore b/.prettierignore index 49284af952..4eadc69b4d 100644 --- a/.prettierignore +++ b/.prettierignore @@ -2,3 +2,6 @@ coverage/ localData/ localMetadata/ junit/ + +# Website fixtures: their exact bytes are asserted as ETags in tests +tests/functional/aws-node-sdk/test/object/websiteFiles/ diff --git a/CLAUDE.md b/CLAUDE.md index f96dcb50db..8c2eca5bfa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -97,23 +97,23 @@ implementations: ### Data Backends (`S3DATA`) -| Backend | Port | Description | -|---------|------|-------------| -| `file` | 9991 | Local filesystem via `dataserver.js` | -| `multiple` | - | Multi-backend gateway (AWS S3, Azure, GCP, Sproxyd) | -| `mem` | - | In-memory (testing only) | +| Backend | Port | Description | +| ---------- | ---- | --------------------------------------------------- | +| `file` | 9991 | Local filesystem via `dataserver.js` | +| `multiple` | - | Multi-backend gateway (AWS S3, Azure, GCP, Sproxyd) | +| `mem` | - | In-memory (testing only) | `scality` is an alias for `multiple`. With `multiple`, objects route to backends defined in `locationConfig.json` based on location constraints. ### Metadata Backends (`S3METADATA`) -| Backend | Port | Description | -|---------|------|-------------| -| `file` (default) | 9990 | Local LevelDB via `mdserver.js` | -| `scality` | 9000 | External bucketd service (production Scality RING) | -| `mongodb` | 27017+ | MongoDB replica set | -| `mem` | - | In-memory (testing only) | +| Backend | Port | Description | +| ---------------- | ------ | ------------------------------------------ | +| `file` (default) | 9990 | Local LevelDB via `mdserver.js` | +| `scality` | 9000 | External bucketd (production Scality RING) | +| `mongodb` | 27017+ | MongoDB replica set | +| `mem` | - | In-memory (testing only) | **file vs scality**: The `file` backend runs a self-contained metadata server (`mdserver.js`) for development. The `scality` backend connects to external @@ -122,20 +122,20 @@ HA deployments. ### Auth Backends (`S3VAULT`) -| Backend | Port | Description | -|---------|------|-------------| -| `mem` (default) | - | In-memory accounts from `conf/authdata.json` | -| `vault` | 8500 | External Vault IAM service (vaultd) | +| Backend | Port | Description | +| --------------- | ---- | -------------------------------------------- | +| `mem` (default) | - | In-memory accounts from `conf/authdata.json` | +| `vault` | 8500 | External Vault IAM service (vaultd) | ### KMS Backends (`S3KMS`) -| Backend | Description | -|---------|-------------| +| Backend | Description | +| ---------------- | ---------------------------- | | `file` (default) | Local file-based key storage | -| `mem` | In-memory (testing only) | -| `kmip` | External KMIP server | -| `aws` | AWS KMS | -| `scality` | Scality KMS | +| `mem` | In-memory (testing only) | +| `kmip` | External KMIP server | +| `aws` | AWS KMS | +| `scality` | Scality KMS | ### Path Configuration diff --git a/Healthchecks.md b/Healthchecks.md index 3950c8bf29..95372bf23e 100644 --- a/Healthchecks.md +++ b/Healthchecks.md @@ -6,19 +6,19 @@ response with HTTP code - 200 OK - Server is up and running +Server is up and running - 500 Internal Server error - Server is experiencing an Internal Error +Server is experiencing an Internal Error - 400 Bad Request - Bad Request due to unsupported HTTP methods +Bad Request due to unsupported HTTP methods - 403 Forbidden - Request is not allowed due to IP restriction +Request is not allowed due to IP restriction ## Stats @@ -53,12 +53,12 @@ returned. This is accomplished by retrieving the 6 keys that represent the 6 five-second intervals. As Redis does not have a performant RANGE query, the list of keys are built manually as follows -* Take current timestamp +- Take current timestamp -* Build each key by subtracting the interval from the timestamp (5 seconds) +- Build each key by subtracting the interval from the timestamp (5 seconds) -* Total keys for each metric (total requests, 500s etc.) is TTL / interval - 30/5 = 6 +- Total keys for each metric (total requests, 500s etc.) is TTL / interval + 30/5 = 6 Note: When Redis is queried, results from non-existent keys are set to 0. diff --git a/TESTING.md b/TESTING.md index 68b8ff7a08..94f48527e0 100644 --- a/TESTING.md +++ b/TESTING.md @@ -10,26 +10,26 @@ ### Features tested - Authentication - - Building signature - - Checking timestamp - - Canonicalization - - Error Handling + - Building signature + - Checking timestamp + - Canonicalization + - Error Handling - Bucket Metadata API - - GET, PUT, DELETE Bucket Metadata + - GET, PUT, DELETE Bucket Metadata - s3 API - - GET Service - - GET, PUT, DELETE, HEAD Object - - GET, PUT, DELETE, HEAD Bucket - - ACL's - - Bucket Policies - - Lifecycle - - Range requests - - Multi-part upload + - GET Service + - GET, PUT, DELETE, HEAD Object + - GET, PUT, DELETE, HEAD Bucket + - ACL's + - Bucket Policies + - Lifecycle + - Range requests + - Multi-part upload - Routes - - GET, PUT, PUTRAW, DELETE, HEAD for objects and buckets + - GET, PUT, PUTRAW, DELETE, HEAD for objects and buckets ## Functional Tests diff --git a/bin/metrics_server.js b/bin/metrics_server.js index 752eb911f8..a44cf517fc 100755 --- a/bin/metrics_server.js +++ b/bin/metrics_server.js @@ -1,18 +1,11 @@ #!/usr/bin/env node 'use strict'; -const { - startWSManagementClient, - startPushConnectionHealthCheckServer, -} = require('../lib/management/push'); +const { startWSManagementClient, startPushConnectionHealthCheckServer } = require('../lib/management/push'); const logger = require('../lib/utilities/logger'); -const { - PUSH_ENDPOINT: pushEndpoint, - INSTANCE_ID: instanceId, - MANAGEMENT_TOKEN: managementToken, -} = process.env; +const { PUSH_ENDPOINT: pushEndpoint, INSTANCE_ID: instanceId, MANAGEMENT_TOKEN: managementToken } = process.env; if (!pushEndpoint) { logger.error('missing push endpoint env var'); diff --git a/bin/search_bucket.js b/bin/search_bucket.js index 14a14b0378..1d566a0724 100755 --- a/bin/search_bucket.js +++ b/bin/search_bucket.js @@ -9,15 +9,7 @@ const http = require('http'); const https = require('https'); const logger = require('../lib/utilities/logger'); -function _performSearch(host, - port, - bucketName, - query, - listVersions, - accessKey, - secretKey, - sessionToken, - verbose, ssl) { +function _performSearch(host, port, bucketName, query, listVersions, accessKey, secretKey, sessionToken, verbose, ssl) { const escapedSearch = encodeURIComponent(query); const options = { host, @@ -88,12 +80,13 @@ function searchBucket() { .option('-h, --host ', 'Host of the server') .option('-p, --port ', 'Port of the server') .option('-s', '--ssl', 'Enable ssl') - .option('-l, --list-versions', 'List all versions of the objects that meet the search query, ' + - 'otherwise only list the latest version') + .option( + '-l, --list-versions', + 'List all versions of the objects that meet the search query, ' + 'otherwise only list the latest version', + ) .option('-v, --verbose') .parse(process.argv); - const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } = - commander; + const { host, port, accessKey, secretKey, sessionToken, bucket, query, listVersions, verbose, ssl } = commander; if (!host || !port || !accessKey || !secretKey || !bucket || !query) { logger.error('missing parameter'); @@ -101,8 +94,7 @@ function searchBucket() { process.exit(1); } - _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose, - ssl); + _performSearch(host, port, bucket, query, listVersions, accessKey, secretKey, sessionToken, verbose, ssl); } searchBucket(); diff --git a/bin/secure_channel_proxy.js b/bin/secure_channel_proxy.js index 165bf94c74..8b235f7fe0 100755 --- a/bin/secure_channel_proxy.js +++ b/bin/secure_channel_proxy.js @@ -1,18 +1,11 @@ #!/usr/bin/env node 'use strict'; -const { - startWSManagementClient, - startPushConnectionHealthCheckServer, -} = require('../lib/management/push'); +const { startWSManagementClient, startPushConnectionHealthCheckServer } = require('../lib/management/push'); const logger = require('../lib/utilities/logger'); -const { - PUSH_ENDPOINT: pushEndpoint, - INSTANCE_ID: instanceId, - MANAGEMENT_TOKEN: managementToken, -} = process.env; +const { PUSH_ENDPOINT: pushEndpoint, INSTANCE_ID: instanceId, MANAGEMENT_TOKEN: managementToken } = process.env; if (!pushEndpoint) { logger.error('missing push endpoint env var'); diff --git a/codecov.yml b/codecov.yml index ee3ed8cfa2..b1e83b5500 100644 --- a/codecov.yml +++ b/codecov.yml @@ -10,8 +10,8 @@ parsers: enable_partials: yes comment: - layout: newheader, reach, files, components, diff, flags # show component info in the PR comment - hide_comment_details: true # hide the comment details (e.g. coverage targets) in the PR comment + layout: newheader, reach, files, components, diff, flags # show component info in the PR comment + hide_comment_details: true # hide the comment details (e.g. coverage targets) in the PR comment # Setting coverage targets coverage: @@ -29,11 +29,11 @@ coverage: target: 80% component_management: - default_rules: # default rules that will be inherited by all components + default_rules: # default rules that will be inherited by all components statuses: [] flag_management: - default_rules: # the rules that will be followed for any flag added, generally + default_rules: # the rules that will be followed for any flag added, generally carryforward: true statuses: [] diff --git a/conf/authdata.json b/conf/authdata.json index 8cd34421c3..5f58ebde09 100644 --- a/conf/authdata.json +++ b/conf/authdata.json @@ -1,56 +1,69 @@ { - "accounts": [{ - "name": "Bart", - "email": "sampleaccount1@sampling.com", - "arn": "arn:aws:iam::123456789012:root", - "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", - "shortid": "123456789012", - "keys": [{ - "access": "accessKey1", - "secret": "verySecretKey1" - }] - }, { - "name": "Lisa", - "email": "sampleaccount2@sampling.com", - "arn": "arn:aws:iam::123456789013:root", - "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", - "shortid": "123456789013", - "keys": [{ - "access": "accessKey2", - "secret": "verySecretKey2" - }] - }, - { - "name": "Clueso", - "email": "inspector@clueso.info", - "arn": "arn:aws:iam::123456789014:root", - "canonicalID": "http://acs.zenko.io/accounts/service/clueso", - "shortid": "123456789014", - "keys": [{ - "access": "cluesoKey1", - "secret": "cluesoSecretKey1" - }] - }, - { - "name": "Replication", - "email": "inspector@replication.info", - "arn": "arn:aws:iam::123456789015:root", - "canonicalID": "http://acs.zenko.io/accounts/service/replication", - "shortid": "123456789015", - "keys": [{ - "access": "replicationKey1", - "secret": "replicationSecretKey1" - }] - }, - { - "name": "Lifecycle", - "email": "inspector@lifecycle.info", - "arn": "arn:aws:iam::123456789016:root", - "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle", - "shortid": "123456789016", - "keys": [{ - "access": "lifecycleKey1", - "secret": "lifecycleSecretKey1" - }] - }] + "accounts": [ + { + "name": "Bart", + "email": "sampleaccount1@sampling.com", + "arn": "arn:aws:iam::123456789012:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", + "shortid": "123456789012", + "keys": [ + { + "access": "accessKey1", + "secret": "verySecretKey1" + } + ] + }, + { + "name": "Lisa", + "email": "sampleaccount2@sampling.com", + "arn": "arn:aws:iam::123456789013:root", + "canonicalID": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf", + "shortid": "123456789013", + "keys": [ + { + "access": "accessKey2", + "secret": "verySecretKey2" + } + ] + }, + { + "name": "Clueso", + "email": "inspector@clueso.info", + "arn": "arn:aws:iam::123456789014:root", + "canonicalID": "http://acs.zenko.io/accounts/service/clueso", + "shortid": "123456789014", + "keys": [ + { + "access": "cluesoKey1", + "secret": "cluesoSecretKey1" + } + ] + }, + { + "name": "Replication", + "email": "inspector@replication.info", + "arn": "arn:aws:iam::123456789015:root", + "canonicalID": "http://acs.zenko.io/accounts/service/replication", + "shortid": "123456789015", + "keys": [ + { + "access": "replicationKey1", + "secret": "replicationSecretKey1" + } + ] + }, + { + "name": "Lifecycle", + "email": "inspector@lifecycle.info", + "arn": "arn:aws:iam::123456789016:root", + "canonicalID": "http://acs.zenko.io/accounts/service/lifecycle", + "shortid": "123456789016", + "keys": [ + { + "access": "lifecycleKey1", + "secret": "lifecycleSecretKey1" + } + ] + } + ] } diff --git a/dataserver.js b/dataserver.js index f1321422fb..f6bcabe481 100644 --- a/dataserver.js +++ b/dataserver.js @@ -14,9 +14,10 @@ process.on('uncaughtException', err => { process.exit(1); }); -if (config.backends.data === 'file' || - (config.backends.data === 'multiple' && - config.backends.metadata !== 'scality')) { +if ( + config.backends.data === 'file' || + (config.backends.data === 'multiple' && config.backends.metadata !== 'scality') +) { const dataServer = new arsenal.network.rest.RESTServer({ bindAddress: config.dataDaemon.bindAddress, port: config.dataDaemon.port, @@ -30,8 +31,7 @@ if (config.backends.data === 'file' || }); dataServer.setup(err => { if (err) { - logger.error('Error initializing REST data server', - { error: err }); + logger.error('Error initializing REST data server', { error: err }); return; } dataServer.start(); diff --git a/docs/OBJECT_LOCK_TEST_PLAN.md b/docs/OBJECT_LOCK_TEST_PLAN.md index 2c97ae5666..5072451d08 100644 --- a/docs/OBJECT_LOCK_TEST_PLAN.md +++ b/docs/OBJECT_LOCK_TEST_PLAN.md @@ -21,7 +21,7 @@ the new API actions. ### putBucket tests - passing option to enable object lock updates bucket metadata and enables - bucket versioning + bucket versioning ### putBucketVersioning tests @@ -43,17 +43,17 @@ the new API actions. ### initiateMultipartUpload tests - mpu object initiated with retention information should include retention - information + information ### putObjectLockConfiguration tests - putting configuration as non-bucket-owner user returns AccessDenied error - disabling object lock on bucket created with object lock returns error - enabling object lock on bucket created without object lock returns - InvalidBucketState error + InvalidBucketState error - enabling object lock with token on bucket created without object lock succeeds - putting valid object lock configuration when bucket does not have object - lock enabled returns error (InvalidRequest?) + lock enabled returns error (InvalidRequest?) - putting valid object lock configuration updates bucket metadata - putting invalid object lock configuration returns error - ObjectLockEnabled !== "Enabled" @@ -66,35 +66,35 @@ the new API actions. - getting configuration as non-bucket-owner user returns AccessDenied error - getting configuration when none is set returns - ObjectLockConfigurationNotFoundError error + ObjectLockConfigurationNotFoundError error - getting configuration returns correct object lock configuration for bucket ### putObjectRetention - putting retention as non-bucket-owner user returns AccessDenied error - putting retention on object in bucket without object lock enabled returns - InvalidRequest error + InvalidRequest error - putting valid retention period updates object metadata ### getObjectRetention - getting retention as non-bucket-owner user returns AccessDenied error - getting retention when none is set returns NoSuchObjectLockConfiguration - error + error - getting retention returns correct object retention period ### putObjectLegalHold - putting legal hold as non-bucket-owner user returns AccessDenied error - putting legal hold on object in bucket without object lock enabled returns - InvalidRequest error + InvalidRequest error - putting valid legal hold updates object metadata ### getObjectLegalHold - getting legal hold as non-bucket-owner user returns AccessDenied error - getting legal hold when none is set returns NoSuchObjectLockConfiguration - error + error - getting legal hold returns correct object legal hold ## End to End Tests @@ -102,22 +102,22 @@ the new API actions. ### Scenarios - Create bucket with object lock enabled. Put object. Put object lock - configuration. Put another object. + configuration. Put another object. - Ensure object put before configuration does not have retention period set - Ensure object put after configuration does have retention period set - Create bucket without object lock. Put object. Enable object lock with token - and put object lock configuration. Put another object. + and put object lock configuration. Put another object. - Ensure object put before configuration does not have retention period set - Ensure object put after configuration does have retention period set - Create bucket with object lock enabled and put configuration with COMPLIANCE - mode. Put object. + mode. Put object. - Ensure object cannot be deleted (returns AccessDenied error). - Ensure object cannot be overwritten. - Create bucket with object lock enabled and put configuration with GOVERNANCE - mode. Put object. + mode. Put object. - Ensure user without permission cannot delete object - Ensure user without permission cannot overwrite object - Ensure user with permission can delete object @@ -126,29 +126,29 @@ the new API actions. - Ensure user with permission cannot shorten retention period - Create bucket with object lock enabled and put configuration. Edit bucket - metadata so retention period is expired. Put object. + metadata so retention period is expired. Put object. - Ensure object can be deleted. - Ensure object can be overwritten. - Create bucket with object lock enabled and put configuration. Edit bucket - metadata so retention period is expired. Put object. Put new retention - period on object. + metadata so retention period is expired. Put object. Put new retention + period on object. - Ensure object cannot be deleted. - Ensure object cannot be overwritten. - Create bucket with object locked enabled and put configuration. Put object. - Edit object metadata so retention period is past expiration. + Edit object metadata so retention period is past expiration. - Ensure object can be deleted. - Ensure object can be overwritten. - Create bucket with object lock enabled and put configuration. Edit bucket - metadata so retention period is expired. Put object. Put legal hold - on object. + metadata so retention period is expired. Put object. Put legal hold + on object. - Ensure object cannot be deleted. - Ensure object cannot be overwritten. - Create bucket with object lock enabled and put configuration. Put object. - Check object retention. Change bucket object lock configuration. + Check object retention. Change bucket object lock configuration. - Ensure object retention period has not changed with bucket configuration. - Create bucket with object lock enabled. Put object with legal hold. @@ -156,6 +156,6 @@ the new API actions. - Ensure object cannot be overwritten. - Create bucket with object lock enabled. Put object with legal hold. Remove - legal hold. + legal hold. - Ensure object can be deleted. - Ensure object can be overwritten. diff --git a/docs/RELEASE.md b/docs/RELEASE.md index ddfb379180..87ab37070e 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -5,8 +5,8 @@ Docker images are hosted on [ghcri.io](https://github.com/orgs/scality/packages). CloudServer has a few images there: -* Cloudserver container image: ghcr.io/scality/cloudserver -* Dashboard oras image: ghcr.io/scality/cloudserver/cloudserver-dashboards +- Cloudserver container image: ghcr.io/scality/cloudserver +- Dashboard oras image: ghcr.io/scality/cloudserver/cloudserver-dashboards With every CI build, the CI will push images, tagging the content with the developer branch's short SHA-1 commit hash. @@ -26,38 +26,36 @@ docker pull ghcr.io/scality/cloudserver: To release a production image: -* Create a PR to bump the package version : +- Create a PR to bump the package version : update Cloudserver's `package.json` by bumping it to the relevant next version in a new PR. Per example if the last released version was `8.4.7`, the next version would be `8.4.8`. - ```js - { - "name": "cloudserver", - "version": "8.4.8", <--- Here - [...] - } - ``` - -* Review & merge the PR - -* Trigger the release workflow on GitHub - - * Go to the [**Actions** tab on GitHub](https://github.com/scality/cloudserver/actions) - * Select the `release` workflow from the list - * Click on **Run workflow** (manual dispatch) - * Enter the new tag (e.g., `8.4.8`) in the input field - * Start the workflow - - This workflow will create the tag and push the Docker images. - - This should be done as soon as the PR is merged, - so that the tag is put on the "version bump" commit. - -* Release the release version on Jira - - * Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page) - * Create a next version - * Name: `[next version]`, in this example `8.4.9` - * Click `...` and select `Release` on the recently released version (`8.4.8`) - * Fill in the field to move incomplete version to the next one + ```js + { + "name": "cloudserver", + "version": "8.4.8", <--- Here + [...] + } + ``` + +- Review & merge the PR + +- Trigger the release workflow on GitHub + - Go to the [**Actions** tab on GitHub](https://github.com/scality/cloudserver/actions) + - Select the `release` workflow from the list + - Click on **Run workflow** (manual dispatch) + - Enter the new tag (e.g., `8.4.8`) in the input field + - Start the workflow + + This workflow will create the tag and push the Docker images. + + This should be done as soon as the PR is merged, + so that the tag is put on the "version bump" commit. + +- Release the release version on Jira + - Go to the [CloudServer release page](https://scality.atlassian.net/projects/CLDSRV?selectedItem=com.atlassian.jira.jira-projects-plugin:release-page) + - Create a next version + - Name: `[next version]`, in this example `8.4.9` + - Click `...` and select `Release` on the recently released version (`8.4.8`) + - Fill in the field to move incomplete version to the next one diff --git a/eslint.config.mjs b/eslint.config.mjs index d647d0f2af..5bc5a1c391 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -1,77 +1,81 @@ -import mocha from "eslint-plugin-mocha"; -import promise from "eslint-plugin-promise"; -import n from "eslint-plugin-n"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; -import js from "@eslint/js"; -import { FlatCompat } from "@eslint/eslintrc"; +import mocha from 'eslint-plugin-mocha'; +import promise from 'eslint-plugin-promise'; +import n from 'eslint-plugin-n'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import js from '@eslint/js'; +import { FlatCompat } from '@eslint/eslintrc'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const compat = new FlatCompat({ baseDirectory: __dirname, recommendedConfig: js.configs.recommended, - allConfig: js.configs.all + allConfig: js.configs.all, }); -export default [...compat.extends('@scality/scality'), { - plugins: { - mocha, - promise, - n, - }, +export default [ + ...compat.extends('@scality/scality'), + { + plugins: { + mocha, + promise, + n, + }, - languageOptions: { - ecmaVersion: 2021, - sourceType: "script", - }, + languageOptions: { + ecmaVersion: 2021, + sourceType: 'script', + }, - rules: { - "import/extensions": "off", - "lines-around-directive": "off", - "no-underscore-dangle": "off", - "indent": "off", - "object-curly-newline": "off", - "operator-linebreak": "off", - "function-paren-newline": "off", - "import/newline-after-import": "off", - "prefer-destructuring": "off", - "implicit-arrow-linebreak": "off", - "no-bitwise": "off", - "dot-location": "off", - "comma-dangle": "off", - "no-undef-init": "off", - "global-require": "off", - "import/no-dynamic-require": "off", - "class-methods-use-this": "off", - "no-plusplus": "off", - "no-else-return": "off", - "object-property-newline": "off", - "import/order": "off", - "no-continue": "off", - "no-tabs": "off", - "lines-between-class-members": "off", - "prefer-spread": "off", - "no-lonely-if": "off", - "no-useless-escape": "off", - "no-restricted-globals": "off", - "no-buffer-constructor": "off", - "import/no-extraneous-dependencies": "off", - "space-unary-ops": "off", - "no-useless-return": "off", - "no-unexpected-multiline": "off", - "no-mixed-operators": "off", - "newline-per-chained-call": "off", - "operator-assignment": "off", - "spaced-comment": "off", - "comma-style": "off", - "no-restricted-properties": "off", - "new-parens": "off", - "no-multi-spaces": "off", - "quote-props": "off", - "mocha/no-exclusive-tests": "error", - "no-redeclare": ["error", { "builtinGlobals": false }], - "promise/prefer-await-to-then": "warn", - "n/callback-return": "warn", + rules: { + 'import/extensions': 'off', + 'lines-around-directive': 'off', + 'no-underscore-dangle': 'off', + indent: 'off', + 'object-curly-newline': 'off', + 'operator-linebreak': 'off', + 'function-paren-newline': 'off', + 'import/newline-after-import': 'off', + 'prefer-destructuring': 'off', + 'implicit-arrow-linebreak': 'off', + 'no-bitwise': 'off', + 'dot-location': 'off', + 'comma-dangle': 'off', + 'no-undef-init': 'off', + 'global-require': 'off', + 'import/no-dynamic-require': 'off', + 'class-methods-use-this': 'off', + 'no-plusplus': 'off', + 'no-else-return': 'off', + 'object-property-newline': 'off', + 'import/order': 'off', + 'no-continue': 'off', + 'no-tabs': 'off', + 'lines-between-class-members': 'off', + 'prefer-spread': 'off', + 'no-lonely-if': 'off', + 'no-useless-escape': 'off', + 'no-restricted-globals': 'off', + 'no-buffer-constructor': 'off', + 'import/no-extraneous-dependencies': 'off', + 'space-unary-ops': 'off', + 'no-useless-return': 'off', + 'no-unexpected-multiline': 'off', + 'no-mixed-operators': 'off', + 'newline-per-chained-call': 'off', + 'operator-assignment': 'off', + 'spaced-comment': 'off', + 'comma-style': 'off', + 'no-restricted-properties': 'off', + 'new-parens': 'off', + 'no-multi-spaces': 'off', + 'quote-props': 'off', + 'max-len': 'off', + 'mocha/no-exclusive-tests': 'error', + 'no-redeclare': ['error', { builtinGlobals: false }], + 'promise/prefer-await-to-then': 'warn', + 'n/callback-return': 'warn', + }, }, -}]; +]; diff --git a/examples/node-md-search.js b/examples/node-md-search.js index d660e44fb1..54068795fc 100644 --- a/examples/node-md-search.js +++ b/examples/node-md-search.js @@ -32,12 +32,13 @@ command.middlewareStack.add( { step: 'build', name: 'addSearchParameter', - priority: 'high' - } + priority: 'high', + }, ); // Send command and handle response -s3Client.send(command) +s3Client + .send(command) .then(data => { process.stdout.write(`Result ${JSON.stringify(data)}`); }) diff --git a/lib/api/apiUtils/authorization/permissionChecks.js b/lib/api/apiUtils/authorization/permissionChecks.js index d82a1bc392..2459d552d4 100644 --- a/lib/api/apiUtils/authorization/permissionChecks.js +++ b/lib/api/apiUtils/authorization/permissionChecks.js @@ -16,8 +16,7 @@ const { } = constants; // whitelist buckets to allow public read on objects -const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS - ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : []; +const publicReadBuckets = process.env.ALLOW_PUBLIC_READ_BUCKETS ? process.env.ALLOW_PUBLIC_READ_BUCKETS.split(',') : []; function getServiceAccountProperties(canonicalID) { const canonicalIDArray = canonicalID.split('/'); @@ -65,12 +64,9 @@ const checkBucketPolicyResult = Object.freeze({ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { // Same logic applies on the Versioned APIs, so let's simplify it. - let requestTypeParsed = requestType.endsWith('Version') ? - requestType.slice(0, 'Version'.length * -1) : requestType; - requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ? - 'objectPut' : requestTypeParsed; - const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? - 'objectPut' : mainApiCall; + let requestTypeParsed = requestType.endsWith('Version') ? requestType.slice(0, 'Version'.length * -1) : requestType; + requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestTypeParsed) ? 'objectPut' : requestTypeParsed; + const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? 'objectPut' : mainApiCall; if (bucket.getOwner() === canonicalID) { return true; } @@ -87,64 +83,66 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { const bucketAcl = bucket.getAcl(); if (requestTypeParsed === 'bucketGet' || requestTypeParsed === 'bucketHead') { - if (bucketAcl.Canned === 'public-read' - || bucketAcl.Canned === 'public-read-write' - || (bucketAcl.Canned === 'authenticated-read' - && canonicalID !== publicId)) { + if ( + bucketAcl.Canned === 'public-read' || + bucketAcl.Canned === 'public-read-write' || + (bucketAcl.Canned === 'authenticated-read' && canonicalID !== publicId) + ) { return true; - } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.READ.indexOf(canonicalID) > -1) { + } else if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.READ.indexOf(canonicalID) > -1) { return true; - } else if (bucketAcl.READ.indexOf(publicId) > -1 - || (bucketAcl.READ.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.READ.indexOf(publicId) > -1 || + (bucketAcl.READ.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'bucketGetACL') { - if ((bucketAcl.Canned === 'log-delivery-write' - && canonicalID === logId) - || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.READ_ACP.indexOf(canonicalID) > -1) { + if ( + (bucketAcl.Canned === 'log-delivery-write' && canonicalID === logId) || + bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || + bucketAcl.READ_ACP.indexOf(canonicalID) > -1 + ) { return true; - } else if (bucketAcl.READ_ACP.indexOf(publicId) > -1 - || (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.READ_ACP.indexOf(publicId) > -1 || + (bucketAcl.READ_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'bucketPutACL') { - if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) { + if (bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || bucketAcl.WRITE_ACP.indexOf(canonicalID) > -1) { return true; - } else if (bucketAcl.WRITE_ACP.indexOf(publicId) > -1 - || (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.WRITE_ACP.indexOf(publicId) > -1 || + (bucketAcl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'objectDelete' || requestTypeParsed === 'objectPut') { - if (bucketAcl.Canned === 'public-read-write' - || bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 - || bucketAcl.WRITE.indexOf(canonicalID) > -1) { + if ( + bucketAcl.Canned === 'public-read-write' || + bucketAcl.FULL_CONTROL.indexOf(canonicalID) > -1 || + bucketAcl.WRITE.indexOf(canonicalID) > -1 + ) { return true; - } else if (bucketAcl.WRITE.indexOf(publicId) > -1 - || (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || bucketAcl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + bucketAcl.WRITE.indexOf(publicId) > -1 || + (bucketAcl.WRITE.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (bucketAcl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + bucketAcl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } @@ -154,20 +152,28 @@ function checkBucketAcls(bucket, requestType, canonicalID, mainApiCall) { // objectPutACL, objectGetACL, objectHead or objectGet, the bucket // authorization check should just return true so can move on to check // rights at the object level. - return (requestTypeParsed === 'objectPutACL' || requestTypeParsed === 'objectGetACL' - || requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); + return ( + requestTypeParsed === 'objectPutACL' || + requestTypeParsed === 'objectGetACL' || + requestTypeParsed === 'objectGet' || + requestTypeParsed === 'objectHead' + ); } -function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIsNotUser, - isUserUnauthenticated, mainApiCall) { +function checkObjectAcls( + bucket, + objectMD, + requestType, + canonicalID, + requesterIsNotUser, + isUserUnauthenticated, + mainApiCall, +) { const bucketOwner = bucket.getOwner(); - const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ? - 'objectPut' : requestType; - const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? - 'objectPut' : mainApiCall; + const requestTypeParsed = actionsToConsiderAsObjectPut.includes(requestType) ? 'objectPut' : requestType; + const parsedMainApiCall = actionsToConsiderAsObjectPut.includes(mainApiCall) ? 'objectPut' : mainApiCall; // acls don't distinguish between users and accounts, so both should be allowed - if (bucketOwnerActions.includes(requestTypeParsed) - && (bucketOwner === canonicalID)) { + if (bucketOwnerActions.includes(requestTypeParsed) && bucketOwner === canonicalID) { return true; } if (objectMD['owner-id'] === canonicalID) { @@ -176,8 +182,10 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs // Backward compatibility if (parsedMainApiCall === 'objectGet') { - if ((isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id'])) - && requestTypeParsed === 'objectGetTagging') { + if ( + (isUserUnauthenticated || (requesterIsNotUser && bucketOwner === objectMD['owner-id'])) && + requestTypeParsed === 'objectGetTagging' + ) { return true; } } @@ -187,25 +195,26 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs } if (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead') { - if (objectMD.acl.Canned === 'public-read' - || objectMD.acl.Canned === 'public-read-write' - || (objectMD.acl.Canned === 'authenticated-read' - && canonicalID !== publicId)) { + if ( + objectMD.acl.Canned === 'public-read' || + objectMD.acl.Canned === 'public-read-write' || + (objectMD.acl.Canned === 'authenticated-read' && canonicalID !== publicId) + ) { return true; - } else if (objectMD.acl.Canned === 'bucket-owner-read' - && bucketOwner === canonicalID) { + } else if (objectMD.acl.Canned === 'bucket-owner-read' && bucketOwner === canonicalID) { return true; - } else if ((objectMD.acl.Canned === 'bucket-owner-full-control' - && bucketOwner === canonicalID) - || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 - || objectMD.acl.READ.indexOf(canonicalID) > -1) { + } else if ( + (objectMD.acl.Canned === 'bucket-owner-full-control' && bucketOwner === canonicalID) || + objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || + objectMD.acl.READ.indexOf(canonicalID) > -1 + ) { return true; - } else if (objectMD.acl.READ.indexOf(publicId) > -1 - || (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + objectMD.acl.READ.indexOf(publicId) > -1 || + (objectMD.acl.READ.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } @@ -217,33 +226,35 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs } if (requestTypeParsed === 'objectPutACL') { - if ((objectMD.acl.Canned === 'bucket-owner-full-control' - && bucketOwner === canonicalID) - || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 - || objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1) { + if ( + (objectMD.acl.Canned === 'bucket-owner-full-control' && bucketOwner === canonicalID) || + objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || + objectMD.acl.WRITE_ACP.indexOf(canonicalID) > -1 + ) { return true; - } else if (objectMD.acl.WRITE_ACP.indexOf(publicId) > -1 - || (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + objectMD.acl.WRITE_ACP.indexOf(publicId) > -1 || + (objectMD.acl.WRITE_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } if (requestTypeParsed === 'objectGetACL') { - if ((objectMD.acl.Canned === 'bucket-owner-full-control' - && bucketOwner === canonicalID) - || objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 - || objectMD.acl.READ_ACP.indexOf(canonicalID) > -1) { + if ( + (objectMD.acl.Canned === 'bucket-owner-full-control' && bucketOwner === canonicalID) || + objectMD.acl.FULL_CONTROL.indexOf(canonicalID) > -1 || + objectMD.acl.READ_ACP.indexOf(canonicalID) > -1 + ) { return true; - } else if (objectMD.acl.READ_ACP.indexOf(publicId) > -1 - || (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 - && canonicalID !== publicId) - || objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1) { + } else if ( + objectMD.acl.READ_ACP.indexOf(publicId) > -1 || + (objectMD.acl.READ_ACP.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + (objectMD.acl.FULL_CONTROL.indexOf(allAuthedUsersId) > -1 && canonicalID !== publicId) || + objectMD.acl.FULL_CONTROL.indexOf(publicId) > -1 + ) { return true; } } @@ -251,9 +262,10 @@ function checkObjectAcls(bucket, objectMD, requestType, canonicalID, requesterIs // allow public reads on buckets that are whitelisted for anonymous reads // TODO: remove this after bucket policies are implemented const bucketAcl = bucket.getAcl(); - const allowPublicReads = publicReadBuckets.includes(bucket.getName()) - && bucketAcl.Canned === 'public-read' - && (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); + const allowPublicReads = + publicReadBuckets.includes(bucket.getName()) && + bucketAcl.Canned === 'public-read' && + (requestTypeParsed === 'objectGet' || requestTypeParsed === 'objectHead'); if (allowPublicReads) { return true; } @@ -289,7 +301,7 @@ function _getAccountId(arn) { } function _isAccountId(principal) { - return (principal.length === 12 && /^\d+$/.test(principal)); + return principal.length === 12 && /^\d+$/.test(principal); } /** @@ -329,15 +341,17 @@ function _checkCrossAccount(requesterARN, requesterCanonicalID, bucketOwnerCanon // Vault returns ARNs like 'arn:aws:iam::123456789012:/accountName/' for root accounts // with an empty resource type (missing 'user/' prefix) if (!_isRootUser(requesterARN)) { - return bucketOwnerCanonicalID === requesterCanonicalID ? - checkPrincipalResult.OK : checkPrincipalResult.CROSS_ACCOUNT_OK; + return bucketOwnerCanonicalID === requesterCanonicalID + ? checkPrincipalResult.OK + : checkPrincipalResult.CROSS_ACCOUNT_OK; } return checkPrincipalResult.OK; } function _checkPrincipalWildcard(requestARN, requesterCanonicalID, bucketOwnerCanonicalID) { - if (requestARN === undefined) { // User in unauthenticated (anonymous request) + if (requestARN === undefined) { + // User in unauthenticated (anonymous request) return checkPrincipalResult.OK; } @@ -349,7 +363,8 @@ function _checkPrincipalAWS(principal, requesterARN, requesterCanonicalID, bucke return _checkPrincipalWildcard(requesterARN, requesterCanonicalID, bucketOwnerCanonicalID); } - if (requesterARN === undefined) { // User in unauthenticated (anonymous request) + if (requesterARN === undefined) { + // User in unauthenticated (anonymous request) return checkPrincipalResult.KO; } @@ -373,7 +388,8 @@ function _checkPrincipalCanonicalUser(principal, requesterARN, requesterCanonica return _checkPrincipalWildcard(requesterARN, requesterCanonicalID, bucketOwnerCanonicalID); } - if (requesterARN === undefined) { // User in unauthenticated (anonymous request) + if (requesterARN === undefined) { + // User in unauthenticated (anonymous request) return checkPrincipalResult.KO; } @@ -411,13 +427,15 @@ function _checkPrincipals(canonicalID, arn, principal, bucketOwnerCanonicalID) { } if (principal.CanonicalUser) { - return _findBestPrincipalMatch(principal.CanonicalUser, - p => _checkPrincipalCanonicalUser(p, arn, canonicalID, bucketOwnerCanonicalID)); + return _findBestPrincipalMatch(principal.CanonicalUser, p => + _checkPrincipalCanonicalUser(p, arn, canonicalID, bucketOwnerCanonicalID), + ); } if (principal.AWS) { - return _findBestPrincipalMatch(principal.AWS, - p => _checkPrincipalAWS(p, arn, canonicalID, bucketOwnerCanonicalID)); + return _findBestPrincipalMatch(principal.AWS, p => + _checkPrincipalAWS(p, arn, canonicalID, bucketOwnerCanonicalID), + ); } return checkPrincipalResult.KO; @@ -456,11 +474,30 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l const ip = request ? requestUtils.getClientIp(request, config) : undefined; const isSecure = request ? requestUtils.getHttpProtocolSecurity(request, config) : undefined; - const requestContext = request ? new RequestContext(request.headers, request.query, - request.bucketName, request.objectKey, ip, - isSecure, request.resourceType, 's3', null, null, - null, null, null, null, null, null, null, null, null, - request.objectLockRetentionDays) : undefined; + const requestContext = request + ? new RequestContext( + request.headers, + request.query, + request.bucketName, + request.objectKey, + ip, + isSecure, + request.resourceType, + 's3', + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + request.objectLockRetentionDays, + ) + : undefined; while (copiedStatement.length > 0) { const s = copiedStatement[0]; @@ -470,34 +507,34 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l const conditionsMatch = _checkBucketPolicyConditions(requestContext, s.Condition, log); const ok = principalMatch === checkPrincipalResult.OK && actionMatch && resourceMatch && conditionsMatch; - const okCross = principalMatch === checkPrincipalResult.CROSS_ACCOUNT_OK - && actionMatch && resourceMatch && conditionsMatch; + const okCross = + principalMatch === checkPrincipalResult.CROSS_ACCOUNT_OK && actionMatch && resourceMatch && conditionsMatch; switch (permission) { - case checkBucketPolicyResult.DEFAULT_DENY: - if ((ok || okCross) && s.Effect === 'Deny') { - return checkBucketPolicyResult.EXPLICIT_DENY; - } else if (ok && s.Effect === 'Allow') { - permission = checkBucketPolicyResult.ALLOW; - } else if (okCross && s.Effect === 'Allow') { - permission = checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW; - } - break; - case checkBucketPolicyResult.EXPLICIT_DENY: - return checkBucketPolicyResult.EXPLICIT_DENY; - case checkBucketPolicyResult.ALLOW: - if ((ok || okCross) && s.Effect === 'Deny') { - return checkBucketPolicyResult.EXPLICIT_DENY; - } - break; - case checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW: - if ((ok || okCross) && s.Effect === 'Deny') { + case checkBucketPolicyResult.DEFAULT_DENY: + if ((ok || okCross) && s.Effect === 'Deny') { + return checkBucketPolicyResult.EXPLICIT_DENY; + } else if (ok && s.Effect === 'Allow') { + permission = checkBucketPolicyResult.ALLOW; + } else if (okCross && s.Effect === 'Allow') { + permission = checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW; + } + break; + case checkBucketPolicyResult.EXPLICIT_DENY: return checkBucketPolicyResult.EXPLICIT_DENY; - } else if (ok && s.Effect === 'Allow') { - permission = checkBucketPolicyResult.ALLOW; - } - break; - default: // Needed for the linter, should be unreachable. - break; + case checkBucketPolicyResult.ALLOW: + if ((ok || okCross) && s.Effect === 'Deny') { + return checkBucketPolicyResult.EXPLICIT_DENY; + } + break; + case checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW: + if ((ok || okCross) && s.Effect === 'Deny') { + return checkBucketPolicyResult.EXPLICIT_DENY; + } else if (ok && s.Effect === 'Allow') { + permission = checkBucketPolicyResult.ALLOW; + } + break; + default: // Needed for the linter, should be unreachable. + break; } copiedStatement = copiedStatement.splice(1); @@ -505,8 +542,18 @@ function checkBucketPolicy(policy, requestType, canonicalID, arn, bucketOwner, l return permission; } -function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, log, - request, aclPermission, results, actionImplicitDenies) { +function processBucketPolicy( + requestType, + bucket, + canonicalID, + arn, + bucketOwner, + log, + request, + aclPermission, + results, + actionImplicitDenies, +) { const bucketPolicy = bucket.getBucketPolicy(); let processedResult = results[requestType]; let aclRequired = false; @@ -514,15 +561,25 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, processedResult = actionImplicitDenies[requestType] === false && aclPermission; aclRequired = true; } else { - const bucketPolicyPermission = checkBucketPolicy(bucketPolicy, requestType, canonicalID, arn, - bucketOwner, log, request, actionImplicitDenies); + const bucketPolicyPermission = checkBucketPolicy( + bucketPolicy, + requestType, + canonicalID, + arn, + bucketOwner, + log, + request, + actionImplicitDenies, + ); if (bucketPolicyPermission === checkBucketPolicyResult.EXPLICIT_DENY) { processedResult = false; } else if (bucketPolicyPermission === checkBucketPolicyResult.ALLOW) { processedResult = true; - } else if (bucketPolicyPermission === checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW - && actionImplicitDenies[requestType] === false) { + } else if ( + bucketPolicyPermission === checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW && + actionImplicitDenies[requestType] === false + ) { // If the bucket policy is cross account, only return true if Vault also returned an explicit allow. processedResult = true; } else { @@ -533,8 +590,16 @@ function processBucketPolicy(requestType, bucket, canonicalID, arn, bucketOwner, return { allowed: processedResult, aclRequired }; } -function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, log, request, - actionImplicitDeniesInput = {}, isWebsite = false) { +function isBucketAuthorized( + bucket, + requestTypesInput, + canonicalID, + authInfo, + log, + request, + actionImplicitDeniesInput = {}, + isWebsite = false, +) { const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput]; const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput; const mainApiCall = requestTypes[0]; @@ -553,7 +618,7 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo arn = authInfo.getArn(); } // if the bucket owner is an account, users should not have default access - if ((bucket.getOwner() === canonicalID) && requesterIsNotUser || isServiceAccount(canonicalID)) { + if ((bucket.getOwner() === canonicalID && requesterIsNotUser) || isServiceAccount(canonicalID)) { results[_requestType] = actionImplicitDenies[_requestType] === false; return results[_requestType]; } @@ -567,8 +632,18 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo _requestType = 'objectGet'; actionImplicitDenies.objectGet = actionImplicitDenies.objectGet || false; } - const { allowed, aclRequired } = processBucketPolicy(_requestType, bucket, canonicalID, arn, - bucket.getOwner(), log, request, aclPermission, results, actionImplicitDenies); + const { allowed, aclRequired } = processBucketPolicy( + _requestType, + bucket, + canonicalID, + arn, + bucket.getOwner(), + log, + request, + aclPermission, + results, + actionImplicitDenies, + ); if (aclRequired && request?.serverAccessLog) { // eslint-disable-next-line no-param-reassign request.serverAccessLog.aclRequired = 'Yes'; @@ -577,8 +652,15 @@ function isBucketAuthorized(bucket, requestTypesInput, canonicalID, authInfo, lo }); } -function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, authInfo, actionImplicitDeniesInput = {}, - log, request) { +function evaluateBucketPolicyWithIAM( + bucket, + requestTypesInput, + canonicalID, + authInfo, + actionImplicitDeniesInput = {}, + log, + request, +) { const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput]; const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput; const results = {}; @@ -590,14 +672,33 @@ function evaluateBucketPolicyWithIAM(bucket, requestTypesInput, canonicalID, aut if (authInfo) { arn = authInfo.getArn(); } - const { allowed } = processBucketPolicy(_requestType, bucket, canonicalID, arn, bucket.getOwner(), log, - request, true, results, actionImplicitDenies); + const { allowed } = processBucketPolicy( + _requestType, + bucket, + canonicalID, + arn, + bucket.getOwner(), + log, + request, + true, + results, + actionImplicitDenies, + ); return allowed; }); } -function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authInfo, log, request, - actionImplicitDeniesInput = {}, isWebsite = false) { +function isObjAuthorized( + bucket, + objectMD, + requestTypesInput, + canonicalID, + authInfo, + log, + request, + actionImplicitDeniesInput = {}, + isWebsite = false, +) { const requestTypes = Array.isArray(requestTypesInput) ? requestTypesInput : [requestTypesInput]; const actionImplicitDenies = !actionImplicitDeniesInput ? {} : actionImplicitDeniesInput; const results = {}; @@ -606,8 +707,7 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI // By default, all missing actions are defined as allowed from IAM, to be // backward compatible actionImplicitDenies[_requestType] = actionImplicitDenies[_requestType] || false; - const parsedMethodName = _requestType.endsWith('Version') - ? _requestType.slice(0, -7) : _requestType; + const parsedMethodName = _requestType.endsWith('Version') ? _requestType.slice(0, -7) : _requestType; const bucketOwner = bucket.getOwner(); if (!objectMD) { // check bucket has read access @@ -616,12 +716,22 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI if (actionsToConsiderAsObjectPut.includes(_requestType)) { permission = 'objectPut'; } - results[_requestType] = isBucketAuthorized(bucket, permission, canonicalID, authInfo, log, request, - actionImplicitDenies, isWebsite); + results[_requestType] = isBucketAuthorized( + bucket, + permission, + canonicalID, + authInfo, + log, + request, + actionImplicitDenies, + isWebsite, + ); // User is already authorized on the bucket for FULL_CONTROL or WRITE or // bucket has canned ACL public-read-write - if ((parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') - && results[_requestType] === false) { + if ( + (parsedMethodName === 'objectPut' || parsedMethodName === 'objectDelete') && + results[_requestType] === false + ) { results[_requestType] = actionImplicitDenies[_requestType] === false; } return results[_requestType]; @@ -634,7 +744,7 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI arn = authInfo.getArn(); isUserUnauthenticated = arn === undefined; } - if (objectMD['owner-id'] === canonicalID && requesterIsNotUser || isServiceAccount(canonicalID)) { + if ((objectMD['owner-id'] === canonicalID && requesterIsNotUser) || isServiceAccount(canonicalID)) { results[_requestType] = actionImplicitDenies[_requestType] === false; return results[_requestType]; } @@ -642,16 +752,31 @@ function isObjAuthorized(bucket, objectMD, requestTypesInput, canonicalID, authI // - requesttype is included in bucketOwnerActions and // - account is the bucket owner // - requester is account, not user - if (bucketOwnerActions.includes(parsedMethodName) - && (bucketOwner === canonicalID) - && requesterIsNotUser) { + if (bucketOwnerActions.includes(parsedMethodName) && bucketOwner === canonicalID && requesterIsNotUser) { results[_requestType] = actionImplicitDenies[_requestType] === false; return results[_requestType]; } - const aclPermission = checkObjectAcls(bucket, objectMD, parsedMethodName, - canonicalID, requesterIsNotUser, isUserUnauthenticated, mainApiCall); - const { allowed, aclRequired } = processBucketPolicy(_requestType, bucket, canonicalID, arn, bucketOwner, - log, request, aclPermission, results, actionImplicitDenies); + const aclPermission = checkObjectAcls( + bucket, + objectMD, + parsedMethodName, + canonicalID, + requesterIsNotUser, + isUserUnauthenticated, + mainApiCall, + ); + const { allowed, aclRequired } = processBucketPolicy( + _requestType, + bucket, + canonicalID, + arn, + bucketOwner, + log, + request, + aclPermission, + results, + actionImplicitDenies, + ); if (aclRequired && request?.serverAccessLog) { // eslint-disable-next-line no-param-reassign request.serverAccessLog.aclRequired = 'Yes'; @@ -752,8 +877,8 @@ function validatePolicyConditions(policy) { for (const conditionOperator of conditionOperators) { const conditionKey = Object.keys(s.Condition[conditionOperator])[0]; const conditionValue = s.Condition[conditionOperator][conditionKey]; - const validCondition = validConditions.find(validCondition => - validCondition.conditionKey === conditionKey + const validCondition = validConditions.find( + validCondition => validCondition.conditionKey === conditionKey, ); // AWS returns does not return an error if the condition starts with 'aws:' // so we reproduce this behaviour @@ -772,7 +897,6 @@ function validatePolicyConditions(policy) { return null; } - /** isLifecycleSession - check if it is the Lifecycle assumed role session arn. * @param {string} arn - Amazon resource name - example: * arn:aws:sts::257038443293:assumed-role/rolename/backbeat-lifecycle @@ -791,9 +915,9 @@ function isLifecycleSession(arn) { const resourceType = resourceNames[0]; const sessionName = resourceNames[resourceNames.length - 1]; - return (service === 'sts' - && resourceType === assumedRoleArnResourceType - && sessionName === backbeatLifecycleSessionName); + return ( + service === 'sts' && resourceType === assumedRoleArnResourceType && sessionName === backbeatLifecycleSessionName + ); } module.exports = { diff --git a/lib/api/apiUtils/authorization/prepareRequestContexts.js b/lib/api/apiUtils/authorization/prepareRequestContexts.js index 4fcf504aeb..1b7d58b610 100644 --- a/lib/api/apiUtils/authorization/prepareRequestContexts.js +++ b/lib/api/apiUtils/authorization/prepareRequestContexts.js @@ -18,9 +18,13 @@ const apiMethodWithVersion = { }; function isHeaderAcl(headers) { - return headers['x-amz-grant-read'] || headers['x-amz-grant-read-acp'] || - headers['x-amz-grant-write-acp'] || headers['x-amz-grant-full-control'] || - headers['x-amz-acl']; + return ( + headers['x-amz-grant-read'] || + headers['x-amz-grant-read-acp'] || + headers['x-amz-grant-write-acp'] || + headers['x-amz-grant-full-control'] || + headers['x-amz-acl'] + ); } /** @@ -32,8 +36,7 @@ function isHeaderAcl(headers) { * @param {string} sourceVersionId - value of sourceVersionId if copy request * @return {RequestContext []} array of requestContexts */ -function prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId) { +function prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId) { // if multiObjectDelete request, we want to authenticate // before parsing the post body and creating multiple requestContexts // so send null as requestContexts to Vault to avoid authorization @@ -48,17 +51,23 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, const isSecure = requestUtils.getHttpProtocolSecurity(request, config); function generateRequestContext(apiMethod) { - return new RequestContext(request.headers, - request.query, request.bucketName, request.objectKey, - ip, isSecure, apiMethod, 's3'); + return new RequestContext( + request.headers, + request.query, + request.bucketName, + request.objectKey, + ip, + isSecure, + apiMethod, + 's3', + ); } if (apiMethod === 'bucketPut') { return null; } - if (apiMethodWithVersion[apiMethod] && request.query && - request.query.versionId) { + if (apiMethodWithVersion[apiMethod] && request.query && request.query.versionId) { apiMethodAfterVersionCheck = `${apiMethod}Version`; } else { apiMethodAfterVersionCheck = apiMethod; @@ -75,181 +84,153 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, // In the API, we then ignore these authorization results, and we can use // any information returned, e.g., the quota. const requestContextMultiObjectDelete = generateRequestContext('objectDelete'); - requestContexts.push(requestContextMultiObjectDelete); - } else if (apiMethodAfterVersionCheck === 'objectCopy' - || apiMethodAfterVersionCheck === 'objectPutCopyPart') { - const objectGetAction = sourceVersionId ? 'objectGetVersion' : - 'objectGet'; - const reqQuery = Object.assign({}, request.query, - { versionId: sourceVersionId }); - const getRequestContext = new RequestContext(request.headers, - reqQuery, sourceBucket, sourceObject, - ip, isSecure, - objectGetAction, 's3'); + requestContexts.push(requestContextMultiObjectDelete); + } else if (apiMethodAfterVersionCheck === 'objectCopy' || apiMethodAfterVersionCheck === 'objectPutCopyPart') { + const objectGetAction = sourceVersionId ? 'objectGetVersion' : 'objectGet'; + const reqQuery = Object.assign({}, request.query, { versionId: sourceVersionId }); + const getRequestContext = new RequestContext( + request.headers, + reqQuery, + sourceBucket, + sourceObject, + ip, + isSecure, + objectGetAction, + 's3', + ); const putRequestContext = generateRequestContext('objectPut'); requestContexts.push(getRequestContext, putRequestContext); if (apiMethodAfterVersionCheck === 'objectCopy') { // if tagging directive is COPY, "s3:PutObjectTagging" don't need // to be included in the list of permitted actions in IAM policy - if (request.headers['x-amz-tagging'] && - request.headers['x-amz-tagging-directive'] === 'REPLACE') { - const putTaggingRequestContext = - generateRequestContext('objectPutTagging'); + if (request.headers['x-amz-tagging'] && request.headers['x-amz-tagging-directive'] === 'REPLACE') { + const putTaggingRequestContext = generateRequestContext('objectPutTagging'); requestContexts.push(putTaggingRequestContext); } if (isHeaderAcl(request.headers)) { - const putAclRequestContext = - generateRequestContext('objectPutACL'); + const putAclRequestContext = generateRequestContext('objectPutACL'); requestContexts.push(putAclRequestContext); } } - } else if (apiMethodAfterVersionCheck === 'objectGet' - || apiMethodAfterVersionCheck === 'objectGetVersion') { - const objectGetTaggingAction = (request.query && - request.query.versionId) ? 'objectGetTaggingVersion' : - 'objectGetTagging'; + } else if (apiMethodAfterVersionCheck === 'objectGet' || apiMethodAfterVersionCheck === 'objectGetVersion') { + const objectGetTaggingAction = + request.query && request.query.versionId ? 'objectGetTaggingVersion' : 'objectGetTagging'; if (request.headers['x-amz-version-id']) { const objectGetVersionAction = 'objectGetVersion'; - const getVersionResourceVersion = - generateRequestContext(objectGetVersionAction); + const getVersionResourceVersion = generateRequestContext(objectGetVersionAction); requestContexts.push(getVersionResourceVersion); } - const getRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); - const getTaggingRequestContext = - generateRequestContext(objectGetTaggingAction); + const getRequestContext = generateRequestContext(apiMethodAfterVersionCheck); + const getTaggingRequestContext = generateRequestContext(objectGetTaggingAction); requestContexts.push(getRequestContext, getTaggingRequestContext); } else if (apiMethodAfterVersionCheck === 'objectGetTagging') { const objectGetTaggingAction = 'objectGetTagging'; - const getTaggingResourceVersion = - generateRequestContext(objectGetTaggingAction); + const getTaggingResourceVersion = generateRequestContext(objectGetTaggingAction); requestContexts.push(getTaggingResourceVersion); if (request.headers['x-amz-version-id']) { const objectGetTaggingVersionAction = 'objectGetTaggingVersion'; - const getTaggingVersionResourceVersion = - generateRequestContext(objectGetTaggingVersionAction); + const getTaggingVersionResourceVersion = generateRequestContext(objectGetTaggingVersionAction); requestContexts.push(getTaggingVersionResourceVersion); } } else if (apiMethodAfterVersionCheck === 'objectHead') { const objectHeadAction = 'objectHead'; - const headObjectAction = - generateRequestContext(objectHeadAction); + const headObjectAction = generateRequestContext(objectHeadAction); requestContexts.push(headObjectAction); if (request.headers['x-amz-version-id']) { const objectHeadVersionAction = 'objectGetVersion'; - const headObjectVersion = - generateRequestContext(objectHeadVersionAction); + const headObjectVersion = generateRequestContext(objectHeadVersionAction); requestContexts.push(headObjectVersion); } if (request.headers['x-amz-scal-archive-info']) { - const coldStatus = - generateRequestContext('objectGetArchiveInfo'); + const coldStatus = generateRequestContext('objectGetArchiveInfo'); requestContexts.push(coldStatus); } } else if (apiMethodAfterVersionCheck === 'objectPutTagging') { - const putObjectTaggingRequestContext = - generateRequestContext('objectPutTagging'); + const putObjectTaggingRequestContext = generateRequestContext('objectPutTagging'); requestContexts.push(putObjectTaggingRequestContext); if (request.headers['x-amz-version-id']) { - const putObjectVersionRequestContext = - generateRequestContext('objectPutTaggingVersion'); + const putObjectVersionRequestContext = generateRequestContext('objectPutTaggingVersion'); requestContexts.push(putObjectVersionRequestContext); } } else if (apiMethodAfterVersionCheck === 'objectPut') { // if put object with version - if (request.headers['x-scal-s3-version-id'] || - request.headers['x-scal-s3-version-id'] === '') { - const putVersionRequestContext = - generateRequestContext('objectPutVersion'); + if (request.headers['x-scal-s3-version-id'] || request.headers['x-scal-s3-version-id'] === '') { + const putVersionRequestContext = generateRequestContext('objectPutVersion'); requestContexts.push(putVersionRequestContext); } else { - const putRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const putRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(putRequestContext); // if put object (versioning) with tag set if (request.headers['x-amz-tagging']) { - const putTaggingRequestContext = - generateRequestContext('objectPutTagging'); + const putTaggingRequestContext = generateRequestContext('objectPutTagging'); requestContexts.push(putTaggingRequestContext); } if (['ON', 'OFF'].includes(request.headers['x-amz-object-lock-legal-hold-status'])) { - const putLegalHoldStatusAction = - generateRequestContext('objectPutLegalHold'); + const putLegalHoldStatusAction = generateRequestContext('objectPutLegalHold'); requestContexts.push(putLegalHoldStatusAction); } // if put object (versioning) with ACL if (isHeaderAcl(request.headers)) { - const putAclRequestContext = - generateRequestContext('objectPutACL'); + const putAclRequestContext = generateRequestContext('objectPutACL'); requestContexts.push(putAclRequestContext); } if (request.headers['x-amz-object-lock-mode']) { - const putObjectLockRequestContext = - generateRequestContext('objectPutRetention'); + const putObjectLockRequestContext = generateRequestContext('objectPutRetention'); requestContexts.push(putObjectLockRequestContext); if (hasGovernanceBypassHeader(request.headers)) { - const checkUserGovernanceBypassRequestContext = - generateRequestContext('bypassGovernanceRetention'); + const checkUserGovernanceBypassRequestContext = generateRequestContext('bypassGovernanceRetention'); requestContexts.push(checkUserGovernanceBypassRequestContext); } } if (request.headers['x-amz-version-id']) { - const putObjectVersionRequestContext = - generateRequestContext('objectPutTaggingVersion'); + const putObjectVersionRequestContext = generateRequestContext('objectPutTaggingVersion'); requestContexts.push(putObjectVersionRequestContext); } } - } else if (apiMethodAfterVersionCheck === 'objectPutRetention' || - apiMethodAfterVersionCheck === 'objectPutRetentionVersion') { - const putRetentionRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + } else if ( + apiMethodAfterVersionCheck === 'objectPutRetention' || + apiMethodAfterVersionCheck === 'objectPutRetentionVersion' + ) { + const putRetentionRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(putRetentionRequestContext); if (hasGovernanceBypassHeader(request.headers)) { - const checkUserGovernanceBypassRequestContext = - generateRequestContext('bypassGovernanceRetention'); + const checkUserGovernanceBypassRequestContext = generateRequestContext('bypassGovernanceRetention'); requestContexts.push(checkUserGovernanceBypassRequestContext); } - } else if (apiMethodAfterVersionCheck === 'initiateMultipartUpload' || - apiMethodAfterVersionCheck === 'objectPutPart' || - apiMethodAfterVersionCheck === 'completeMultipartUpload' - ) { - if (request.headers['x-scal-s3-version-id'] || - request.headers['x-scal-s3-version-id'] === '') { - const putVersionRequestContext = - generateRequestContext('objectPutVersion'); + } else if ( + apiMethodAfterVersionCheck === 'initiateMultipartUpload' || + apiMethodAfterVersionCheck === 'objectPutPart' || + apiMethodAfterVersionCheck === 'completeMultipartUpload' + ) { + if (request.headers['x-scal-s3-version-id'] || request.headers['x-scal-s3-version-id'] === '') { + const putVersionRequestContext = generateRequestContext('objectPutVersion'); requestContexts.push(putVersionRequestContext); } else { - const putRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const putRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(putRequestContext); } // if put object (versioning) with ACL if (isHeaderAcl(request.headers)) { - const putAclRequestContext = - generateRequestContext('objectPutACL'); + const putAclRequestContext = generateRequestContext('objectPutACL'); requestContexts.push(putAclRequestContext); } if (request.headers['x-amz-object-lock-mode']) { - const putObjectLockRequestContext = - generateRequestContext('objectPutRetention'); + const putObjectLockRequestContext = generateRequestContext('objectPutRetention'); requestContexts.push(putObjectLockRequestContext); } if (request.headers['x-amz-version-id']) { - const putObjectVersionRequestContext = - generateRequestContext('objectPutTaggingVersion'); + const putObjectVersionRequestContext = generateRequestContext('objectPutTaggingVersion'); requestContexts.push(putObjectVersionRequestContext); } - // AWS only returns an object lock error if a version id - // is specified, else continue to create a delete marker + // AWS only returns an object lock error if a version id + // is specified, else continue to create a delete marker } else if (sourceVersionId && apiMethodAfterVersionCheck === 'objectDeleteVersion') { - const deleteRequestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const deleteRequestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(deleteRequestContext); if (hasGovernanceBypassHeader(request.headers)) { - const checkUserGovernanceBypassRequestContext = - generateRequestContext('bypassGovernanceRetention'); + const checkUserGovernanceBypassRequestContext = generateRequestContext('bypassGovernanceRetention'); requestContexts.push(checkUserGovernanceBypassRequestContext); } } else if (apiMethodAfterVersionCheck === 'bucketGet') { @@ -268,10 +249,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, generateRequestContext('objectGetVersionAttributes'), ); } else { - requestContexts.push( - generateRequestContext('objectGet'), - generateRequestContext('objectGetAttributes'), - ); + requestContexts.push(generateRequestContext('objectGet'), generateRequestContext('objectGetAttributes')); } const attributes = request.headers['x-amz-object-attributes']?.split(',') ?? []; @@ -279,8 +257,7 @@ function prepareRequestContexts(apiMethod, request, sourceBucket, requestContexts.push(generateRequestContext('objectGetAttributesCustom')); } } else { - const requestContext = - generateRequestContext(apiMethodAfterVersionCheck); + const requestContext = generateRequestContext(apiMethodAfterVersionCheck); requestContexts.push(requestContext); } diff --git a/lib/api/apiUtils/authorization/serviceUser.js b/lib/api/apiUtils/authorization/serviceUser.js index c3a125954d..743475b3cc 100644 --- a/lib/api/apiUtils/authorization/serviceUser.js +++ b/lib/api/apiUtils/authorization/serviceUser.js @@ -11,5 +11,5 @@ function isRateLimitServiceUser(authInfo) { } module.exports = { - isRateLimitServiceUser + isRateLimitServiceUser, }; diff --git a/lib/api/apiUtils/authorization/tagConditionKeys.js b/lib/api/apiUtils/authorization/tagConditionKeys.js index 47c09ce0b3..3bce314a11 100644 --- a/lib/api/apiUtils/authorization/tagConditionKeys.js +++ b/lib/api/apiUtils/authorization/tagConditionKeys.js @@ -13,36 +13,36 @@ function makeTagQuery(tags) { } function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, cb) { - async.waterfall([ - next => { - if (request.headers['x-amz-tagging']) { - return next(null, request.headers['x-amz-tagging']); - } - if (request.post && apiMethod === 'objectPutTagging') { - return parseTagXml(request.post, log, (err, tags) => { - if (err) { - log.trace('error parsing request tags'); - return next(err); - } - return next(null, makeTagQuery(tags)); - }); - } - return next(null, null); - }, - (requestTagsQuery, next) => { - const objectKey = request.objectKey; - const bucketName = request.bucketName; - const decodedVidResult = decodeVersionId(request.query); - if (decodedVidResult instanceof Error) { - log.trace('invalid versionId query', { - versionId: request.query.versionId, - error: decodedVidResult, - }); - return next(decodedVidResult); - } - const reqVersionId = decodedVidResult; - return metadata.getObjectMD( - bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => { + async.waterfall( + [ + next => { + if (request.headers['x-amz-tagging']) { + return next(null, request.headers['x-amz-tagging']); + } + if (request.post && apiMethod === 'objectPutTagging') { + return parseTagXml(request.post, log, (err, tags) => { + if (err) { + log.trace('error parsing request tags'); + return next(err); + } + return next(null, makeTagQuery(tags)); + }); + } + return next(null, null); + }, + (requestTagsQuery, next) => { + const objectKey = request.objectKey; + const bucketName = request.bucketName; + const decodedVidResult = decodeVersionId(request.query); + if (decodedVidResult instanceof Error) { + log.trace('invalid versionId query', { + versionId: request.query.versionId, + error: decodedVidResult, + }); + return next(decodedVidResult); + } + const reqVersionId = decodedVidResult; + return metadata.getObjectMD(bucketName, objectKey, { versionId: reqVersionId }, log, (err, objMD) => { if (err) { // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver if (err.NoSuchKey) { @@ -54,24 +54,26 @@ function updateRequestContextsWithTags(request, requestContexts, apiMethod, log, const existingTagsQuery = objMD.tags && makeTagQuery(objMD.tags); return next(null, requestTagsQuery, existingTagsQuery); }); - }, - ], (err, requestTagsQuery, existingTagsQuery) => { - if (err) { - log.trace('error processing tag condition key evaluation'); - return cb(err); - } - // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter - for (const rc of requestContexts) { - rc.setNeedTagEval(true); - if (requestTagsQuery) { - rc.setRequestObjTags(requestTagsQuery); + }, + ], + (err, requestTagsQuery, existingTagsQuery) => { + if (err) { + log.trace('error processing tag condition key evaluation'); + return cb(err); } - if (existingTagsQuery) { - rc.setExistingObjTag(existingTagsQuery); + // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter + for (const rc of requestContexts) { + rc.setNeedTagEval(true); + if (requestTagsQuery) { + rc.setRequestObjTags(requestTagsQuery); + } + if (existingTagsQuery) { + rc.setExistingObjTag(existingTagsQuery); + } } - } - return cb(); - }); + return cb(); + }, + ); } function tagConditionKeyAuth(authorizationResults, request, requestContexts, apiMethod, log, cb) { @@ -86,8 +88,13 @@ function tagConditionKeyAuth(authorizationResults, request, requestContexts, api if (err) { return cb(err); } - return auth.server.doAuth(request, log, - (err, userInfo, authResults) => cb(err, authResults), 's3', requestContexts); + return auth.server.doAuth( + request, + log, + (err, userInfo, authResults) => cb(err, authResults), + 's3', + requestContexts, + ); }); } diff --git a/lib/api/apiUtils/bucket/bucketCors.js b/lib/api/apiUtils/bucket/bucketCors.js index e81c3b1199..b76cce7f9b 100644 --- a/lib/api/apiUtils/bucket/bucketCors.js +++ b/lib/api/apiUtils/bucket/bucketCors.js @@ -26,10 +26,8 @@ const escapeForXml = s3middleware.escapeForXml; */ const customizedErrs = { - numberRules: 'The number of CORS rules should not exceed allowed limit ' + - 'of 100 rules.', - originAndMethodExist: 'Each CORSRule must identify at least one origin ' + - 'and one method.', + numberRules: 'The number of CORS rules should not exceed allowed limit ' + 'of 100 rules.', + originAndMethodExist: 'Each CORSRule must identify at least one origin ' + 'and one method.', }; // Helper validation methods @@ -42,21 +40,20 @@ const _validator = { validateNumberWildcards(string) { const firstIndex = string.indexOf('*'); if (firstIndex !== -1) { - return (string.indexOf('*', firstIndex + 1) === -1); + return string.indexOf('*', firstIndex + 1) === -1; } return true; }, /** _validator.validateID - check value of optional ID - * @param {string[]} id - array containing id string - * @return {(Error|true|undefined)} - Arsenal error on failure, true on - * success, undefined if ID does not exist - */ + * @param {string[]} id - array containing id string + * @return {(Error|true|undefined)} - Arsenal error on failure, true on + * success, undefined if ID does not exist + */ validateID(id) { if (!id) { return undefined; // to indicate ID does not exist } - if (!Array.isArray(id) || id.length !== 1 - || typeof id[0] !== 'string') { + if (!Array.isArray(id) || id.length !== 1 || typeof id[0] !== 'string') { return errors.MalformedXML; } if (id[0] === '') { @@ -65,10 +62,10 @@ const _validator = { return true; }, /** _validator.validateMaxAgeSeconds - check value of optional MaxAgeSeconds - * @param {string[]} seconds - array containing number string - * @return {(Error|parsedValue|undefined)} - Arsenal error on failure, parsed - * value if valid, undefined if MaxAgeSeconds does not exist - */ + * @param {string[]} seconds - array containing number string + * @return {(Error|parsedValue|undefined)} - Arsenal error on failure, parsed + * value if valid, undefined if MaxAgeSeconds does not exist + */ validateMaxAgeSeconds(seconds) { if (!seconds) { return undefined; @@ -87,36 +84,37 @@ const _validator = { return parsedValue; }, /** _validator.validateNumberRules - return if number of rules exceeds 100 - * @param {number} length - array containing number string - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {number} length - array containing number string + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateNumberRules(length) { if (length > 100) { - return errorInstances.InvalidRequest - .customizeDescription(customizedErrs.numberRules); + return errorInstances.InvalidRequest.customizeDescription(customizedErrs.numberRules); } return true; }, /** _validator.validateOriginAndMethodExist - * @param {string[]} allowedMethods - array of AllowedMethod's - * @param {string[]} allowedOrigins - array of AllowedOrigin's - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {string[]} allowedMethods - array of AllowedMethod's + * @param {string[]} allowedOrigins - array of AllowedOrigin's + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateOriginAndMethodExist(allowedMethods, allowedOrigins) { - if (allowedOrigins && allowedMethods && - Array.isArray(allowedOrigins) && - Array.isArray(allowedMethods) && - allowedOrigins.length > 0 && - allowedMethods.length > 0) { + if ( + allowedOrigins && + allowedMethods && + Array.isArray(allowedOrigins) && + Array.isArray(allowedMethods) && + allowedOrigins.length > 0 && + allowedMethods.length > 0 + ) { return true; } - return errorInstances.MalformedXML - .customizeDescription(customizedErrs.originAndMethodExist); + return errorInstances.MalformedXML.customizeDescription(customizedErrs.originAndMethodExist); }, /** _validator.validateMethods - check values of AllowedMethod's - * @param {string[]} methods - array of AllowedMethod's - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {string[]} methods - array of AllowedMethod's + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateMethods(methods) { let invalidMethod; function isValidMethod(method) { @@ -128,17 +126,17 @@ const _validator = { return false; } if (!methods.every(isValidMethod)) { - const errMsg = 'Found unsupported HTTP method in CORS config. ' + - `Unsupported method is "${invalidMethod}"`; + const errMsg = + 'Found unsupported HTTP method in CORS config. ' + `Unsupported method is "${invalidMethod}"`; return errorInstances.InvalidRequest.customizeDescription(errMsg); } return true; }, /** _validator.validateAllowedOriginsOrHeaders - check values - * @param {string[]} elementArr - array of elements to check - * @param {string} typeElement - type of element being checked - * @return {(Error|true)} - Arsenal error on failure, true on success - */ + * @param {string[]} elementArr - array of elements to check + * @param {string} typeElement - type of element being checked + * @return {(Error|true)} - Arsenal error on failure, true on success + */ validateAllowedOriginsOrHeaders(elementArr, typeElement) { for (let i = 0; i < elementArr.length; i++) { const element = elementArr[i]; @@ -146,18 +144,17 @@ const _validator = { return errors.MalformedXML; } if (!this.validateNumberWildcards(element)) { - const errMsg = `${typeElement} "${element}" can not have ` + - 'more than one wildcard.'; + const errMsg = `${typeElement} "${element}" can not have ` + 'more than one wildcard.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } } return true; }, /** _validator.validateAllowedHeaders - check values of AllowedHeader's - * @param {string[]} headers - array of AllowedHeader's - * @return {(Error|true|undefined)} - Arsenal error on failure, true if - * valid, undefined if optional AllowedHeader's do not exist - */ + * @param {string[]} headers - array of AllowedHeader's + * @return {(Error|true|undefined)} - Arsenal error on failure, true if + * valid, undefined if optional AllowedHeader's do not exist + */ validateAllowedHeaders(headers) { if (!headers) { return undefined; // to indicate AllowedHeaders do not exist @@ -165,18 +162,17 @@ const _validator = { if (!Array.isArray(headers) || headers.length === 0) { return errors.MalformedXML; } - const result = - this.validateAllowedOriginsOrHeaders(headers, 'AllowedHeader'); + const result = this.validateAllowedOriginsOrHeaders(headers, 'AllowedHeader'); if (result instanceof Error) { return result; } return true; }, /** _validator.validateExposeHeaders - check values of ExposeHeader's - * @param {string[]} headers - array of ExposeHeader's - * @return {(Error|true|undefined)} - Arsenal error on failure, true if - * valid, undefined if optional ExposeHeader's do not exist - */ + * @param {string[]} headers - array of ExposeHeader's + * @return {(Error|true|undefined)} - Arsenal error on failure, true if + * valid, undefined if optional ExposeHeader's do not exist + */ validateExposeHeaders(headers) { if (!headers) { return undefined; // indicate ExposeHeaders do not exist @@ -190,13 +186,13 @@ const _validator = { return errors.MalformedXML; } if (header.indexOf('*') !== -1) { - const errMsg = `ExposeHeader ${header} contains a wildcard. ` + - 'Wildcards are currently not supported for ExposeHeader.'; + const errMsg = + `ExposeHeader ${header} contains a wildcard. ` + + 'Wildcards are currently not supported for ExposeHeader.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (!/^[A-Za-z0-9-]*$/.test(header)) { - const errMsg = `ExposeHeader ${header} contains invalid ` + - 'character.'; + const errMsg = `ExposeHeader ${header} contains invalid ` + 'character.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } } @@ -205,31 +201,29 @@ const _validator = { }; /** _validateCorsXml - Validate XML, returning an error if any part is not valid -* @param {object[]} rules - CORSRule collection parsed from xml to be validated -* @param {string[]} [rules[].ID] - optional id to identify rule -* @param {string[]} rules[].AllowedMethod - methods allowed for CORS -* @param {string[]} rules[].AllowedOrigin - origins allowed for CORS -* @param {string[]} [rules[].AllowedHeader] - headers allowed in an OPTIONS -* request via the Access-Control-Request-Headers header -* @param {string[]} [rules[].MaxAgeSeconds] - seconds browsers should cache -* OPTIONS response -* @param {string[]} [rules[].ExposeHeader] - headers exposed to applications -* @return {(Error|object)} - return cors object on success; error on failure -*/ + * @param {object[]} rules - CORSRule collection parsed from xml to be validated + * @param {string[]} [rules[].ID] - optional id to identify rule + * @param {string[]} rules[].AllowedMethod - methods allowed for CORS + * @param {string[]} rules[].AllowedOrigin - origins allowed for CORS + * @param {string[]} [rules[].AllowedHeader] - headers allowed in an OPTIONS + * request via the Access-Control-Request-Headers header + * @param {string[]} [rules[].MaxAgeSeconds] - seconds browsers should cache + * OPTIONS response + * @param {string[]} [rules[].ExposeHeader] - headers exposed to applications + * @return {(Error|object)} - return cors object on success; error on failure + */ function _validateCorsXml(rules) { const cors = []; let result; if (rules.length > 100) { - return errorInstances.InvalidRequest - .customizeDescription(customizedErrs.numberRules); + return errorInstances.InvalidRequest.customizeDescription(customizedErrs.numberRules); } for (let i = 0; i < rules.length; i++) { const rule = rules[i]; const corsRule = {}; - result = _validator.validateOriginAndMethodExist(rule.AllowedMethod, - rule.AllowedOrigin); + result = _validator.validateOriginAndMethodExist(rule.AllowedMethod, rule.AllowedOrigin); if (result instanceof Error) { return result; } @@ -240,8 +234,7 @@ function _validateCorsXml(rules) { } corsRule.allowedMethods = rule.AllowedMethod; - result = _validator.validateAllowedOriginsOrHeaders(rule.AllowedOrigin, - 'AllowedOrigin'); + result = _validator.validateAllowedOriginsOrHeaders(rule.AllowedOrigin, 'AllowedOrigin'); if (result instanceof Error) { return result; } @@ -281,12 +274,12 @@ function _validateCorsXml(rules) { } /** parseCorsXml - Parse and validate xml body, returning cors object on success -* @param {string} xml - xml body to parse and validate -* @param {object} log - Werelogs logger -* @param {function} cb - callback to server -* @return {undefined} - calls callback with cors object on success, error on -* failure -*/ + * @param {string} xml - xml body to parse and validate + * @param {object} log - Werelogs logger + * @param {function} cb - callback to server + * @return {undefined} - calls callback with cors object on success, error on + * failure + */ function parseCorsXml(xml, log, cb) { parseString(xml, (err, result) => { if (err) { @@ -298,15 +291,17 @@ function parseCorsXml(xml, log, cb) { return cb(errors.MalformedXML); } - if (!result || !result.CORSConfiguration || + if ( + !result || + !result.CORSConfiguration || !result.CORSConfiguration.CORSRule || - !Array.isArray(result.CORSConfiguration.CORSRule)) { + !Array.isArray(result.CORSConfiguration.CORSRule) + ) { const errMsg = 'Invalid cors configuration xml'; return cb(errorInstances.MalformedXML.customizeDescription(errMsg)); } - const validationRes = - _validateCorsXml(result.CORSConfiguration.CORSRule); + const validationRes = _validateCorsXml(result.CORSConfiguration.CORSRule); if (validationRes instanceof Error) { log.debug('xml validation failed', { error: validationRes, @@ -322,18 +317,14 @@ function parseCorsXml(xml, log, cb) { function convertToXml(arrayRules) { const xml = []; - xml.push('', - ''); + xml.push('', ''); arrayRules.forEach(rule => { xml.push(''); - ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders'] - .forEach(key => { + ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders'].forEach(key => { if (rule[key]) { - const element = key.charAt(0).toUpperCase() + - key.slice(1, -1); + const element = key.charAt(0).toUpperCase() + key.slice(1, -1); rule[key].forEach(value => { - xml.push(`<${element}>${escapeForXml(value)}` + - ``); + xml.push(`<${element}>${escapeForXml(value)}` + ``); }); } }); diff --git a/lib/api/apiUtils/bucket/bucketDeletion.js b/lib/api/apiUtils/bucket/bucketDeletion.js index 764727e2d2..67d4d1c36b 100644 --- a/lib/api/apiUtils/bucket/bucketDeletion.js +++ b/lib/api/apiUtils/bucket/bucketDeletion.js @@ -5,15 +5,13 @@ const { errors } = require('arsenal'); const abortMultipartUpload = require('../object/abortMultipartUpload'); const { pushMetric } = require('../../../utapi/utilities'); -const { splitter, oldSplitter, mpuBucketPrefix } = - require('../../../../constants'); +const { splitter, oldSplitter, mpuBucketPrefix } = require('../../../../constants'); const metadata = require('../../../metadata/wrapper'); const kms = require('../../../kms/wrapper'); const deleteUserBucketEntry = require('./deleteUserBucketEntry'); function _deleteMPUbucket(destinationBucketName, log, cb) { - const mpuBucketName = - `${mpuBucketPrefix}${destinationBucketName}`; + const mpuBucketName = `${mpuBucketPrefix}${destinationBucketName}`; return metadata.deleteBucket(mpuBucketName, log, err => { // If the mpu bucket does not exist, just move on // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver @@ -25,23 +23,34 @@ function _deleteMPUbucket(destinationBucketName, log, cb) { } function _deleteOngoingMPUs(authInfo, bucketName, bucketMD, mpus, request, log, cb) { - async.mapLimit(mpus, 1, (mpu, next) => { - const splitterChar = mpu.key.includes(oldSplitter) ? - oldSplitter : splitter; - // `overview${splitter}${objectKey}${splitter}${uploadId} - const [, objectKey, uploadId] = mpu.key.split(splitterChar); - abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, - (err, destBucket, partSizeSum) => { - pushMetric('abortMultipartUpload', log, { - authInfo, - canonicalID: bucketMD.getOwner(), - bucket: bucketName, - keys: [objectKey], - byteLength: partSizeSum, - }); - next(err); - }, request); - }, cb); + async.mapLimit( + mpus, + 1, + (mpu, next) => { + const splitterChar = mpu.key.includes(oldSplitter) ? oldSplitter : splitter; + // `overview${splitter}${objectKey}${splitter}${uploadId} + const [, objectKey, uploadId] = mpu.key.split(splitterChar); + abortMultipartUpload( + authInfo, + bucketName, + objectKey, + uploadId, + log, + (err, destBucket, partSizeSum) => { + pushMetric('abortMultipartUpload', log, { + authInfo, + canonicalID: bucketMD.getOwner(), + bucket: bucketName, + keys: [objectKey], + byteLength: partSizeSum, + }); + next(err); + }, + request, + ); + }, + cb, + ); } /** * deleteBucket - Delete bucket from namespace @@ -60,37 +69,36 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, assert.strictEqual(typeof bucketName, 'string'); assert.strictEqual(typeof canonicalID, 'string'); - return async.waterfall([ - function checkForObjectsStep(next) { - const params = { maxKeys: 1, listingType: 'DelimiterVersions' }; - // We list all the versions as we want to return BucketNotEmpty - // error if there are any versions or delete markers in the bucket. - // Works for non-versioned buckets as well since listing versions - // includes null (non-versioned) objects in the result. - return metadata.listObject(bucketName, params, log, - (err, list) => { + return async.waterfall( + [ + function checkForObjectsStep(next) { + const params = { maxKeys: 1, listingType: 'DelimiterVersions' }; + // We list all the versions as we want to return BucketNotEmpty + // error if there are any versions or delete markers in the bucket. + // Works for non-versioned buckets as well since listing versions + // includes null (non-versioned) objects in the result. + return metadata.listObject(bucketName, params, log, (err, list) => { if (err) { log.error('error from metadata', { error: err }); return next(err); } - const length = (list.Versions ? list.Versions.length : 0) + + const length = + (list.Versions ? list.Versions.length : 0) + (list.DeleteMarkers ? list.DeleteMarkers.length : 0); log.debug('listing result', { length }); if (length) { - log.debug('bucket delete failed', - { error: errors.BucketNotEmpty }); + log.debug('bucket delete failed', { error: errors.BucketNotEmpty }); return next(errors.BucketNotEmpty); } return next(); }); - }, + }, - function deleteMPUbucketStep(next) { - const MPUBucketName = `${mpuBucketPrefix}${bucketName}`; - // check to see if there are any mpu overview objects (so ignore - // any orphaned part objects) - return metadata.listObject(MPUBucketName, { prefix: 'overview' }, - log, (err, objectsListRes) => { + function deleteMPUbucketStep(next) { + const MPUBucketName = `${mpuBucketPrefix}${bucketName}`; + // check to see if there are any mpu overview objects (so ignore + // any orphaned part objects) + return metadata.listObject(MPUBucketName, { prefix: 'overview' }, log, (err, objectsListRes) => { // If no shadow bucket ever created, no ongoing MPU's, so // continue with deletion if (err?.is?.NoSuchBucket) { @@ -101,58 +109,67 @@ function deleteBucket(authInfo, bucketMD, bucketName, canonicalID, request, log, return next(err); } if (objectsListRes.Contents.length) { - return _deleteOngoingMPUs(authInfo, bucketName, - bucketMD, objectsListRes.Contents, request, log, err => { + return _deleteOngoingMPUs( + authInfo, + bucketName, + bucketMD, + objectsListRes.Contents, + request, + log, + err => { if (err) { return next(err); } log.trace('deleting shadow MPU bucket'); return _deleteMPUbucket(bucketName, log, next); - }); + }, + ); } log.trace('deleting shadow MPU bucket'); return _deleteMPUbucket(bucketName, log, next); }); - }, - function addDeleteFlagStep(next) { - log.trace('adding deleted attribute to bucket attributes'); - // Remove transient flag if any so never have both transient - // and deleted flags. - bucketMD.removeTransientFlag(); - bucketMD.addDeletedFlag(); - return metadata.updateBucket(bucketName, bucketMD, log, next); - }, - function deleteUserBucketEntryStep(next) { - log.trace('deleting bucket name from user bucket'); - return deleteUserBucketEntry(bucketName, canonicalID, log, next); - }, - ], - // eslint-disable-next-line prefer-arrow-callback - function actualDeletionStep(err) { - if (err) { - return cb(err); - } - return metadata.deleteBucket(bucketName, log, err => { - log.trace('deleting bucket from metadata'); + }, + function addDeleteFlagStep(next) { + log.trace('adding deleted attribute to bucket attributes'); + // Remove transient flag if any so never have both transient + // and deleted flags. + bucketMD.removeTransientFlag(); + bucketMD.addDeletedFlag(); + return metadata.updateBucket(bucketName, bucketMD, log, next); + }, + function deleteUserBucketEntryStep(next) { + log.trace('deleting bucket name from user bucket'); + return deleteUserBucketEntry(bucketName, canonicalID, log, next); + }, + ], + // eslint-disable-next-line prefer-arrow-callback + function actualDeletionStep(err) { if (err) { return cb(err); } - const serverSideEncryption = bucketMD.getServerSideEncryption(); - const isScalityManagedEncryptionKey = serverSideEncryption && serverSideEncryption.algorithm === 'AES256'; - const isAccountEncryptionEnabled = bucketMD.isAccountEncryptionEnabled(); + return metadata.deleteBucket(bucketName, log, err => { + log.trace('deleting bucket from metadata'); + if (err) { + return cb(err); + } + const serverSideEncryption = bucketMD.getServerSideEncryption(); + const isScalityManagedEncryptionKey = + serverSideEncryption && serverSideEncryption.algorithm === 'AES256'; + const isAccountEncryptionEnabled = bucketMD.isAccountEncryptionEnabled(); - /** - * If all of the following conditions are met, delete the master encryption key: - * - The encryption key is managed by Scality (not externally managed). - * - The encryption is bucket-specific (to prevent deleting default account encryption key). - */ - if (isScalityManagedEncryptionKey && !isAccountEncryptionEnabled) { - const masterKeyId = serverSideEncryption.masterKeyId; - return kms.destroyBucketKey(masterKeyId, log, cb); - } - return cb(); - }); - }); + /** + * If all of the following conditions are met, delete the master encryption key: + * - The encryption key is managed by Scality (not externally managed). + * - The encryption is bucket-specific (to prevent deleting default account encryption key). + */ + if (isScalityManagedEncryptionKey && !isAccountEncryptionEnabled) { + const masterKeyId = serverSideEncryption.masterKeyId; + return kms.destroyBucketKey(masterKeyId, log, cb); + } + return cb(); + }); + }, + ); } module.exports = deleteBucket; diff --git a/lib/api/apiUtils/bucket/bucketEncryption.js b/lib/api/apiUtils/bucket/bucketEncryption.js index 8916e96f1b..e5a567124c 100644 --- a/lib/api/apiUtils/bucket/bucketEncryption.js +++ b/lib/api/apiUtils/bucket/bucketEncryption.js @@ -11,7 +11,7 @@ const { isScalityKmsArn } = require('arsenal/build/lib/network/KMSInterface'); * @property {string} masterKeyId - Key id for the kms key used to encrypt data keys. * @property {string} configuredMasterKeyId - User configured master key id. * @property {boolean} mandatory - Whether a default encryption policy has been enabled. -*/ + */ /** * @callback ServerSideEncryptionInfo~callback @@ -37,9 +37,7 @@ function parseEncryptionXml(xml, log, cb) { return cb(errors.MalformedXML); } - if (!parsed - || !parsed.ServerSideEncryptionConfiguration - || !parsed.ServerSideEncryptionConfiguration.Rule) { + if (!parsed || !parsed.ServerSideEncryptionConfiguration || !parsed.ServerSideEncryptionConfiguration.Rule) { log.trace('error in sse config, invalid ServerSideEncryptionConfiguration section', { method: 'parseEncryptionXml', }); @@ -48,11 +46,13 @@ function parseEncryptionXml(xml, log, cb) { const { Rule } = parsed.ServerSideEncryptionConfiguration; - if (!Array.isArray(Rule) - || Rule.length > 1 - || !Rule[0] - || !Rule[0].ApplyServerSideEncryptionByDefault - || !Rule[0].ApplyServerSideEncryptionByDefault[0]) { + if ( + !Array.isArray(Rule) || + Rule.length > 1 || + !Rule[0] || + !Rule[0].ApplyServerSideEncryptionByDefault || + !Rule[0].ApplyServerSideEncryptionByDefault[0] + ) { log.trace('error in sse config, invalid ApplyServerSideEncryptionByDefault section', { method: 'parseEncryptionXml', }); @@ -84,8 +84,11 @@ function parseEncryptionXml(xml, log, cb) { log.trace('error in sse config, can not specify KMSMasterKeyID when using AES256', { method: 'parseEncryptionXml', }); - return cb(errorInstances.InvalidArgument.customizeDescription( - 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms')); + return cb( + errorInstances.InvalidArgument.customizeDescription( + 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms', + ), + ); } if (!encConfig.KMSMasterKeyID[0] || typeof encConfig.KMSMasterKeyID[0] !== 'string') { @@ -163,15 +166,17 @@ function parseObjectEncryptionHeaders(headers) { if (sseAlgorithm && sseAlgorithm !== 'AES256' && sseAlgorithm !== 'aws:kms') { return { - error: errorInstances.InvalidArgument - .customizeDescription('The encryption method specified is not supported'), + error: errorInstances.InvalidArgument.customizeDescription( + 'The encryption method specified is not supported', + ), }; } if (sseAlgorithm !== 'aws:kms' && configuredMasterKeyId) { return { error: errorInstances.InvalidArgument.customizeDescription( - 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms'), + 'a KMSMasterKeyID is not applicable if the default sse algorithm is not aws:kms', + ), }; } return { objectSSE: hydrateEncryptionConfig(sseAlgorithm, configuredMasterKeyId) }; @@ -185,17 +190,13 @@ function parseObjectEncryptionHeaders(headers) { * @returns {undefined} */ function createDefaultBucketEncryptionMetadata(bucket, log, cb) { - return kms.bucketLevelEncryption( - bucket, - { algorithm: 'AES256', mandatory: false }, - log, - (error, sseConfig) => { - if (error) { - return cb(error); - } - bucket.setServerSideEncryption(sseConfig); - return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig)); - }); + return kms.bucketLevelEncryption(bucket, { algorithm: 'AES256', mandatory: false }, log, (error, sseConfig) => { + if (error) { + return cb(error); + } + bucket.setServerSideEncryption(sseConfig); + return metadata.updateBucket(bucket.getName(), bucket, log, err => cb(err, sseConfig)); + }); } /** diff --git a/lib/api/apiUtils/bucket/bucketShield.js b/lib/api/apiUtils/bucket/bucketShield.js index 483a092ee6..fe07b9eada 100644 --- a/lib/api/apiUtils/bucket/bucketShield.js +++ b/lib/api/apiUtils/bucket/bucketShield.js @@ -9,32 +9,32 @@ const constants = require('../../../../constants'); * @return {boolean} true if the bucket should be shielded, false otherwise */ function bucketShield(bucket, requestType) { - const invisiblyDeleteRequests = constants.bucketOwnerActions.concat( - [ - 'bucketGet', - 'bucketHead', - 'bucketGetACL', - 'objectGet', - 'objectGetACL', - 'objectHead', - 'objectPutACL', - 'objectDelete', - ]); - if (invisiblyDeleteRequests.indexOf(requestType) > -1 && - bucket.hasDeletedFlag()) { + const invisiblyDeleteRequests = constants.bucketOwnerActions.concat([ + 'bucketGet', + 'bucketHead', + 'bucketGetACL', + 'objectGet', + 'objectGetACL', + 'objectHead', + 'objectPutACL', + 'objectDelete', + ]); + if (invisiblyDeleteRequests.indexOf(requestType) > -1 && bucket.hasDeletedFlag()) { invisiblyDelete(bucket.getName(), bucket.getOwner()); return true; } - // If request is initiateMultipartUpload (requestType objectPut), - // objectPut, bucketPutACL or bucketDelete, proceed with request. - // Otherwise return an error to the client - if ((bucket.hasDeletedFlag() || bucket.hasTransientFlag()) && - (requestType !== 'objectPut' && + // If request is initiateMultipartUpload (requestType objectPut), + // objectPut, bucketPutACL or bucketDelete, proceed with request. + // Otherwise return an error to the client + if ( + (bucket.hasDeletedFlag() || bucket.hasTransientFlag()) && + requestType !== 'objectPut' && requestType !== 'initiateMultipartUpload' && requestType !== 'objectPutPart' && requestType !== 'completeMultipartUpload' && requestType !== 'bucketPutACL' && - requestType !== 'bucketDelete')) { + requestType !== 'bucketDelete' + ) { return true; } return false; diff --git a/lib/api/apiUtils/bucket/bucketWebsite.js b/lib/api/apiUtils/bucket/bucketWebsite.js index e723e2387a..e909091bef 100644 --- a/lib/api/apiUtils/bucket/bucketWebsite.js +++ b/lib/api/apiUtils/bucket/bucketWebsite.js @@ -2,8 +2,7 @@ const { parseString } = require('xml2js'); const { errors, errorInstances, s3middleware } = require('arsenal'); const escapeForXml = s3middleware.escapeForXml; -const { WebsiteConfiguration } = - require('arsenal').models.WebsiteConfiguration; +const { WebsiteConfiguration } = require('arsenal').models.WebsiteConfiguration; /* Format of xml request: @@ -30,31 +29,29 @@ const { WebsiteConfiguration } = */ - // Key names of redirect object values to check if are valid strings -const redirectValuesToCheck = ['HostName', 'ReplaceKeyPrefixWith', - 'ReplaceKeyWith']; +const redirectValuesToCheck = ['HostName', 'ReplaceKeyPrefixWith', 'ReplaceKeyWith']; /** Helper function for validating format of parsed xml element -* @param {array} elem - element to check -* @return {boolean} true / false - elem meets expected format -*/ + * @param {array} elem - element to check + * @return {boolean} true / false - elem meets expected format + */ function _isValidElem(elem) { - return (Array.isArray(elem) && elem.length === 1); + return Array.isArray(elem) && elem.length === 1; } /** Check if parsed xml element contains a specified child element -* @param {array} parent - represents xml element to check for child element -* @param {(string|string[])} requiredElem - name of child element(s) -* @param {object} [options] - specify additional options -* @param {boolean} [isList] - indicates if parent is list of children elements, -* used only in conjunction a singular requiredElem argument -* @param {boolean} [checkForAll] - return true only if parent element contains -* all children elements specified in requiredElem; by default, returns true if -* parent element contains at least one -* @param {boolean} [validateParent] - validate format of parent element -* @return {boolean} true / false - if parsed xml element contains child -*/ + * @param {array} parent - represents xml element to check for child element + * @param {(string|string[])} requiredElem - name of child element(s) + * @param {object} [options] - specify additional options + * @param {boolean} [isList] - indicates if parent is list of children elements, + * used only in conjunction a singular requiredElem argument + * @param {boolean} [checkForAll] - return true only if parent element contains + * all children elements specified in requiredElem; by default, returns true if + * parent element contains at least one + * @param {boolean} [validateParent] - validate format of parent element + * @return {boolean} true / false - if parsed xml element contains child + */ function xmlContainsElem(parent, requiredElem, options) { // Non-top level xml is parsed into object in the following manner. @@ -75,8 +72,7 @@ function xmlContainsElem(parent, requiredElem, options) { const checkForAll = options ? options.checkForAll : false; // true by default, validateParent only designated as false when // parent was validated in previous check - const validateParent = (options && options.validateParent !== undefined) ? - options.validateParent : true; + const validateParent = options && options.validateParent !== undefined ? options.validateParent : true; if (validateParent && !_isValidElem(parent)) { return false; @@ -88,8 +84,7 @@ function xmlContainsElem(parent, requiredElem, options) { return requiredElem.some(elem => _isValidElem(parent[0][elem])); } if (isList) { - if (!Array.isArray(parent[0][requiredElem]) || - parent[0][requiredElem].length === 0) { + if (!Array.isArray(parent[0][requiredElem]) || parent[0][requiredElem].length === 0) { return false; } } else { @@ -99,7 +94,6 @@ function xmlContainsElem(parent, requiredElem, options) { return true; } - /** Validate XML, returning an error if any part is not valid * @param {object} parsingResult - object parsed from xml to be validated * @param {object[]} parsingResult.IndexDocument - @@ -146,22 +140,19 @@ function _validateWebsiteConfigXml(parsingResult) { let errMsg; function _isValidString(value) { - return (typeof value === 'string' && value !== ''); + return typeof value === 'string' && value !== ''; } if (!parsingResult.IndexDocument && !parsingResult.RedirectAllRequestsTo) { - errMsg = 'Value for IndexDocument Suffix must be provided if ' + - 'RedirectAllRequestsTo is empty'; + errMsg = 'Value for IndexDocument Suffix must be provided if ' + 'RedirectAllRequestsTo is empty'; return errorInstances.InvalidArgument.customizeDescription(errMsg); } if (parsingResult.RedirectAllRequestsTo) { const parent = parsingResult.RedirectAllRequestsTo; const redirectAllObj = {}; - if (parsingResult.IndexDocument || parsingResult.ErrorDocument || - parsingResult.RoutingRules) { - errMsg = 'RedirectAllRequestsTo cannot be provided in ' + - 'conjunction with other Routing Rules.'; + if (parsingResult.IndexDocument || parsingResult.ErrorDocument || parsingResult.RoutingRules) { + errMsg = 'RedirectAllRequestsTo cannot be provided in ' + 'conjunction with other Routing Rules.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (!xmlContainsElem(parent, 'HostName')) { @@ -174,10 +165,10 @@ function _validateWebsiteConfigXml(parsingResult) { } redirectAllObj.hostName = parent[0].HostName[0]; if (xmlContainsElem(parent, 'Protocol', { validateParent: false })) { - if (parent[0].Protocol[0] !== 'http' && - parent[0].Protocol[0] !== 'https') { - errMsg = 'Invalid protocol, protocol can be http or https. ' + - 'If not defined, the protocol will be selected automatically.'; + if (parent[0].Protocol[0] !== 'http' && parent[0].Protocol[0] !== 'https') { + errMsg = + 'Invalid protocol, protocol can be http or https. ' + + 'If not defined, the protocol will be selected automatically.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } redirectAllObj.protocol = parent[0].Protocol[0]; @@ -190,8 +181,7 @@ function _validateWebsiteConfigXml(parsingResult) { if (!xmlContainsElem(parent, 'Suffix')) { errMsg = 'IndexDocument is not well-formed'; return errorInstances.MalformedXML.customizeDescription(errMsg); - } else if (!_isValidString(parent[0].Suffix[0]) - || parent[0].Suffix[0].indexOf('/') !== -1) { + } else if (!_isValidString(parent[0].Suffix[0]) || parent[0].Suffix[0].indexOf('/') !== -1) { errMsg = 'IndexDocument Suffix is not well-formed'; return errorInstances.InvalidArgument.customizeDescription(errMsg); } @@ -221,8 +211,7 @@ function _validateWebsiteConfigXml(parsingResult) { const rule = parent[0].RoutingRule[i]; const ruleObj = { redirect: {} }; if (!_isValidElem(rule.Redirect)) { - errMsg = 'RoutingRule requires Redirect, which is ' + - 'missing or not well-formed'; + errMsg = 'RoutingRule requires Redirect, which is ' + 'missing or not well-formed'; return errorInstances.MalformedXML.customizeDescription(errMsg); } // Looks like AWS doesn't actually make this check, but AWS @@ -231,28 +220,35 @@ function _validateWebsiteConfigXml(parsingResult) { // elements to know how to implement a redirect for a rule. // http://docs.aws.amazon.com/AmazonS3/latest/API/ // RESTBucketPUTwebsite.html - if (!xmlContainsElem(rule.Redirect, ['Protocol', 'HostName', - 'ReplaceKeyPrefixWith', 'ReplaceKeyWith', 'HttpRedirectCode'], - { validateParent: false })) { - errMsg = 'Redirect must contain at least one of ' + - 'following: Protocol, HostName, ReplaceKeyPrefixWith, ' + - 'ReplaceKeyWith, or HttpRedirectCode element'; + if ( + !xmlContainsElem( + rule.Redirect, + ['Protocol', 'HostName', 'ReplaceKeyPrefixWith', 'ReplaceKeyWith', 'HttpRedirectCode'], + { validateParent: false }, + ) + ) { + errMsg = + 'Redirect must contain at least one of ' + + 'following: Protocol, HostName, ReplaceKeyPrefixWith, ' + + 'ReplaceKeyWith, or HttpRedirectCode element'; return errorInstances.MalformedXML.customizeDescription(errMsg); } if (rule.Redirect[0].Protocol) { - if (!_isValidElem(rule.Redirect[0].Protocol) || - (rule.Redirect[0].Protocol[0] !== 'http' && - rule.Redirect[0].Protocol[0] !== 'https')) { - errMsg = 'Invalid protocol, protocol can be http or ' + - 'https. If not defined, the protocol will be selected ' + - 'automatically.'; + if ( + !_isValidElem(rule.Redirect[0].Protocol) || + (rule.Redirect[0].Protocol[0] !== 'http' && rule.Redirect[0].Protocol[0] !== 'https') + ) { + errMsg = + 'Invalid protocol, protocol can be http or ' + + 'https. If not defined, the protocol will be selected ' + + 'automatically.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } ruleObj.redirect.protocol = rule.Redirect[0].Protocol[0]; } if (rule.Redirect[0].HttpRedirectCode) { - errMsg = 'The provided HTTP redirect code is not valid. ' + - 'It should be a string containing a number.'; + errMsg = + 'The provided HTTP redirect code is not valid. ' + 'It should be a string containing a number.'; if (!_isValidElem(rule.Redirect[0].HttpRedirectCode)) { return errorInstances.MalformedXML.customizeDescription(errMsg); } @@ -261,8 +257,8 @@ function _validateWebsiteConfigXml(parsingResult) { return errorInstances.MalformedXML.customizeDescription(errMsg); } if (!(code > 300 && code < 400)) { - errMsg = `The provided HTTP redirect code (${code}) is ` + - 'not valid. Valid codes are 3XX except 300'; + errMsg = + `The provided HTTP redirect code (${code}) is ` + 'not valid. Valid codes are 3XX except 300'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } ruleObj.redirect.httpRedirectCode = code; @@ -273,57 +269,51 @@ function _validateWebsiteConfigXml(parsingResult) { if (elem) { if (!_isValidElem(elem) || !_isValidString(elem[0])) { errMsg = `Redirect ${elem} is not well-formed`; - return errorInstances.InvalidArgument - .customizeDescription(errMsg); + return errorInstances.InvalidArgument.customizeDescription(errMsg); } - ruleObj.redirect[`${elemName.charAt(0).toLowerCase()}` + - `${elemName.slice(1)}`] = elem[0]; + ruleObj.redirect[`${elemName.charAt(0).toLowerCase()}` + `${elemName.slice(1)}`] = elem[0]; } } - if (xmlContainsElem( - rule.Redirect, - ['ReplaceKeyPrefixWith', 'ReplaceKeyWith'], - { validateParent: false, checkForAll: true })) { - errMsg = 'Redirect must not contain both ReplaceKeyWith ' + - 'and ReplaceKeyPrefixWith'; + if ( + xmlContainsElem(rule.Redirect, ['ReplaceKeyPrefixWith', 'ReplaceKeyWith'], { + validateParent: false, + checkForAll: true, + }) + ) { + errMsg = 'Redirect must not contain both ReplaceKeyWith ' + 'and ReplaceKeyPrefixWith'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (Array.isArray(rule.Condition) && rule.Condition.length === 1) { ruleObj.condition = {}; - if (!xmlContainsElem(rule.Condition, ['KeyPrefixEquals', - 'HttpErrorCodeReturnedEquals'])) { - errMsg = 'Condition is not well-formed. ' + - 'Condition should contain valid KeyPrefixEquals or ' + - 'HttpErrorCodeReturnEquals element.'; + if (!xmlContainsElem(rule.Condition, ['KeyPrefixEquals', 'HttpErrorCodeReturnedEquals'])) { + errMsg = + 'Condition is not well-formed. ' + + 'Condition should contain valid KeyPrefixEquals or ' + + 'HttpErrorCodeReturnEquals element.'; return errorInstances.InvalidRequest.customizeDescription(errMsg); } if (rule.Condition[0].KeyPrefixEquals) { const keyPrefixEquals = rule.Condition[0].KeyPrefixEquals; - if (!_isValidElem(keyPrefixEquals) || - !_isValidString(keyPrefixEquals[0])) { + if (!_isValidElem(keyPrefixEquals) || !_isValidString(keyPrefixEquals[0])) { errMsg = 'Condition KeyPrefixEquals is not well-formed'; - return errorInstances.InvalidArgument - .customizeDescription(errMsg); + return errorInstances.InvalidArgument.customizeDescription(errMsg); } ruleObj.condition.keyPrefixEquals = keyPrefixEquals[0]; } if (rule.Condition[0].HttpErrorCodeReturnedEquals) { - errMsg = 'The provided HTTP error code is not valid. ' + - 'It should be a string containing a number.'; - if (!_isValidElem(rule.Condition[0] - .HttpErrorCodeReturnedEquals)) { + errMsg = + 'The provided HTTP error code is not valid. ' + 'It should be a string containing a number.'; + if (!_isValidElem(rule.Condition[0].HttpErrorCodeReturnedEquals)) { return errorInstances.MalformedXML.customizeDescription(errMsg); } - const code = parseInt(rule.Condition[0] - .HttpErrorCodeReturnedEquals[0], 10); + const code = parseInt(rule.Condition[0].HttpErrorCodeReturnedEquals[0], 10); if (Number.isNaN(code)) { return errorInstances.MalformedXML.customizeDescription(errMsg); } if (!(code > 399 && code < 600)) { - errMsg = `The provided HTTP error code (${code}) is ` + - 'not valid. Valid codes are 4XX or 5XX.'; - return errorInstances.InvalidRequest - .customizeDescription(errMsg); + errMsg = + `The provided HTTP error code (${code}) is ` + 'not valid. Valid codes are 4XX or 5XX.'; + return errorInstances.InvalidRequest.customizeDescription(errMsg); } ruleObj.condition.httpErrorCodeReturnedEquals = code; } @@ -350,8 +340,7 @@ function parseWebsiteConfigXml(xml, log, cb) { return cb(errorInstances.MalformedXML.customizeDescription(errMsg)); } - const validationRes = - _validateWebsiteConfigXml(result.WebsiteConfiguration); + const validationRes = _validateWebsiteConfigXml(result.WebsiteConfiguration); if (validationRes instanceof Error) { log.debug('xml validation failed', { error: validationRes, @@ -375,35 +364,27 @@ function convertToXml(config) { function _pushChildren(obj) { Object.keys(obj).forEach(element => { - const xmlElem = `${element.charAt(0).toUpperCase()}` + - `${element.slice(1)}`; + const xmlElem = `${element.charAt(0).toUpperCase()}` + `${element.slice(1)}`; xml.push(`<${xmlElem}>${escapeForXml(obj[element])}`); }); } - xml.push('', - ''); + xml.push( + '', + '', + ); if (indexDocument) { - xml.push('', - `${escapeForXml(indexDocument)}`, - ''); + xml.push('', `${escapeForXml(indexDocument)}`, ''); } if (errorDocument) { - xml.push('', - `${escapeForXml(errorDocument)}`, - ''); + xml.push('', `${escapeForXml(errorDocument)}`, ''); } if (redirectAllRequestsTo) { xml.push(''); if (redirectAllRequestsTo.hostName) { - xml.push('', - `${escapeForXml(redirectAllRequestsTo.hostName)}`, - ''); + xml.push('', `${escapeForXml(redirectAllRequestsTo.hostName)}`, ''); } if (redirectAllRequestsTo.protocol) { - xml.push('', - `${redirectAllRequestsTo.protocol}`, - ''); + xml.push('', `${redirectAllRequestsTo.protocol}`, ''); } xml.push(''); } diff --git a/lib/api/apiUtils/bucket/checkPreferredLocations.js b/lib/api/apiUtils/bucket/checkPreferredLocations.js index c717c079c5..1c384479f0 100644 --- a/lib/api/apiUtils/bucket/checkPreferredLocations.js +++ b/lib/api/apiUtils/bucket/checkPreferredLocations.js @@ -2,10 +2,10 @@ const { errorInstances } = require('arsenal'); function checkPreferredLocations(location, locationConstraints, log) { const retError = loc => { - const errMsg = 'value of the location you are attempting to set - ' + - `${loc} - is not listed in the locationConstraint config`; - log.trace(`locationConstraint is invalid - ${errMsg}`, - { locationConstraint: loc }); + const errMsg = + 'value of the location you are attempting to set - ' + + `${loc} - is not listed in the locationConstraint config`; + log.trace(`locationConstraint is invalid - ${errMsg}`, { locationConstraint: loc }); return errorInstances.InvalidLocationConstraint.customizeDescription(errMsg); }; if (typeof location === 'string' && !locationConstraints[location]) { diff --git a/lib/api/apiUtils/bucket/createKeyForUserBucket.js b/lib/api/apiUtils/bucket/createKeyForUserBucket.js index 36f23c3706..f4d46e8042 100644 --- a/lib/api/apiUtils/bucket/createKeyForUserBucket.js +++ b/lib/api/apiUtils/bucket/createKeyForUserBucket.js @@ -1,5 +1,4 @@ -function createKeyForUserBucket(canonicalID, - splitter, bucketName) { +function createKeyForUserBucket(canonicalID, splitter, bucketName) { return `${canonicalID}${splitter}${bucketName}`; } diff --git a/lib/api/apiUtils/bucket/deleteUserBucketEntry.js b/lib/api/apiUtils/bucket/deleteUserBucketEntry.js index ff1846e963..8257a67785 100644 --- a/lib/api/apiUtils/bucket/deleteUserBucketEntry.js +++ b/lib/api/apiUtils/bucket/deleteUserBucketEntry.js @@ -1,42 +1,35 @@ const createKeyForUserBucket = require('./createKeyForUserBucket'); -const { usersBucket, oldUsersBucket, splitter, oldSplitter } = - require('../../../../constants'); +const { usersBucket, oldUsersBucket, splitter, oldSplitter } = require('../../../../constants'); const metadata = require('../../../metadata/wrapper'); function deleteUserBucketEntry(bucketName, canonicalID, log, cb) { - log.trace('deleting bucket name from users bucket', { method: - '_deleteUserBucketEntry' }); - const keyForUserBucket = createKeyForUserBucket(canonicalID, splitter, - bucketName); + log.trace('deleting bucket name from users bucket', { method: '_deleteUserBucketEntry' }); + const keyForUserBucket = createKeyForUserBucket(canonicalID, splitter, bucketName); metadata.deleteObjectMD(usersBucket, keyForUserBucket, {}, log, error => { // If the object representing the bucket is not in the // users bucket just continue if (error?.is.NoSuchKey) { return cb(null); - // BACKWARDS COMPATIBILITY: Remove this once no longer - // have old user bucket format + // BACKWARDS COMPATIBILITY: Remove this once no longer + // have old user bucket format } else if (error?.is.NoSuchBucket) { - const keyForUserBucket2 = createKeyForUserBucket(canonicalID, - oldSplitter, bucketName); - return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2, - {}, log, error => { - // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver - if (error && !error.NoSuchKey) { - log.error('from metadata while deleting user bucket', - { error }); - return cb(error); - } - log.trace('deleted bucket from user bucket', - { method: '_deleteUserBucketEntry' }); - return cb(null); - }); + const keyForUserBucket2 = createKeyForUserBucket(canonicalID, oldSplitter, bucketName); + return metadata.deleteObjectMD(oldUsersBucket, keyForUserBucket2, {}, log, error => { + // TODO: move to `.is` once BKTCLT-9 is done and bumped in Cloudserver + if (error && !error.NoSuchKey) { + log.error('from metadata while deleting user bucket', { error }); + return cb(error); + } + log.trace('deleted bucket from user bucket', { method: '_deleteUserBucketEntry' }); + return cb(null); + }); } else if (error) { - log.error('from metadata while deleting user bucket', { error, - method: '_deleteUserBucketEntry' }); + log.error('from metadata while deleting user bucket', { error, method: '_deleteUserBucketEntry' }); return cb(error); } log.trace('deleted bucket from user bucket', { - method: '_deleteUserBucketEntry' }); + method: '_deleteUserBucketEntry', + }); return cb(null); }); } diff --git a/lib/api/apiUtils/bucket/getNotificationConfiguration.js b/lib/api/apiUtils/bucket/getNotificationConfiguration.js index 1f7151235f..2d9ba03d92 100644 --- a/lib/api/apiUtils/bucket/getNotificationConfiguration.js +++ b/lib/api/apiUtils/bucket/getNotificationConfiguration.js @@ -10,8 +10,11 @@ function getNotificationConfiguration(parsedXml) { return notifConfig; } if (!config.bucketNotificationDestinations) { - return { error: errorInstances.InvalidArgument.customizeDescription( - 'Unable to validate the following destination configurations') }; + return { + error: errorInstances.InvalidArgument.customizeDescription( + 'Unable to validate the following destination configurations', + ), + }; } const targets = new Set(config.bucketNotificationDestinations.map(t => t.resource)); const notifConfigTargets = notifConfig.queueConfig.map(t => t.queueArn.split(':')[5]); diff --git a/lib/api/apiUtils/bucket/getReplicationConfiguration.js b/lib/api/apiUtils/bucket/getReplicationConfiguration.js index 1f0e5231d7..9f6bf2c34c 100644 --- a/lib/api/apiUtils/bucket/getReplicationConfiguration.js +++ b/lib/api/apiUtils/bucket/getReplicationConfiguration.js @@ -1,7 +1,6 @@ const config = require('../../../Config').config; const parseXML = require('../../../utilities/parseXML'); -const ReplicationConfiguration = - require('arsenal').models.ReplicationConfiguration; +const ReplicationConfiguration = require('arsenal').models.ReplicationConfiguration; // Handle the steps for returning a valid replication configuration object. function getReplicationConfiguration(xml, log, cb) { diff --git a/lib/api/apiUtils/bucket/invisiblyDelete.js b/lib/api/apiUtils/bucket/invisiblyDelete.js index 5ccf5cd329..2e39cc39a4 100644 --- a/lib/api/apiUtils/bucket/invisiblyDelete.js +++ b/lib/api/apiUtils/bucket/invisiblyDelete.js @@ -15,20 +15,17 @@ function invisiblyDelete(bucketName, canonicalID) { log.trace('deleting bucket with deleted flag invisibly', { bucketName }); return deleteUserBucketEntry(bucketName, canonicalID, log, err => { if (err) { - log.error('error invisibly deleting bucket name from user bucket', - { error: err }); + log.error('error invisibly deleting bucket name from user bucket', { error: err }); return log.end(); } log.trace('deleted bucket name from user bucket'); return metadata.deleteBucket(bucketName, log, error => { - log.trace('deleting bucket from metadata', - { method: 'invisiblyDelete' }); + log.trace('deleting bucket from metadata', { method: 'invisiblyDelete' }); if (error) { log.error('error deleting bucket from metadata', { error }); return log.end(); } - log.trace('invisible deletion of bucket succeeded', - { method: 'invisiblyDelete' }); + log.trace('invisible deletion of bucket succeeded', { method: 'invisiblyDelete' }); return log.end(); }); }); diff --git a/lib/api/apiUtils/bucket/parseWhere.js b/lib/api/apiUtils/bucket/parseWhere.js index 4275f9e1cb..83deccfced 100644 --- a/lib/api/apiUtils/bucket/parseWhere.js +++ b/lib/api/apiUtils/bucket/parseWhere.js @@ -38,7 +38,7 @@ const exprMapper = { '<': '$lt', '>=': '$gte', '<=': '$lte', - 'LIKE': '$regex', + LIKE: '$regex', }; /* @@ -53,18 +53,12 @@ function parseWhere(root) { const e1 = parseWhere(root[operator][0]); const e2 = parseWhere(root[operator][1]); - return { '$and' : [ - e1, - e2, - ] }; + return { $and: [e1, e2] }; } else if (operator === 'OR') { const e1 = parseWhere(root[operator][0]); const e2 = parseWhere(root[operator][1]); - return { '$or' : [ - e1, - e2, - ] }; + return { $or: [e1, e2] }; } const field = root[operator][0]; const value = root[operator][1]; diff --git a/lib/api/apiUtils/bucket/validateReplicationConfig.js b/lib/api/apiUtils/bucket/validateReplicationConfig.js index e0e64df2ca..36807ae14b 100644 --- a/lib/api/apiUtils/bucket/validateReplicationConfig.js +++ b/lib/api/apiUtils/bucket/validateReplicationConfig.js @@ -24,8 +24,7 @@ function validateReplicationConfig(repConfig, bucket) { return true; } const storageClasses = rule.storageClass.split(','); - return storageClasses.some( - site => site.endsWith(':preferred_read')); + return storageClasses.some(site => site.endsWith(':preferred_read')); }); } diff --git a/lib/api/apiUtils/bucket/validateSearch.js b/lib/api/apiUtils/bucket/validateSearch.js index 9a2e33f94c..32d33ed1e3 100644 --- a/lib/api/apiUtils/bucket/validateSearch.js +++ b/lib/api/apiUtils/bucket/validateSearch.js @@ -20,7 +20,7 @@ const sqlConfig = { ], tokenizer: { shouldTokenize: ['(', ')', '=', '!=', '<', '>', '<=', '>=', '<>'], - shouldMatch: ['"', '\'', '`'], + shouldMatch: ['"', "'", '`'], shouldDelimitBy: [' ', '\n', '\r', '\t'], }, }; @@ -39,10 +39,12 @@ function _validateTree(whereClause, possibleAttributes) { _searchTree(node[operator][1]); } else { const field = node[operator][0]; - if (!field.startsWith('tags.') && + if ( + !field.startsWith('tags.') && !possibleAttributes[field] && !field.startsWith('replicationInfo.') && - !field.startsWith('x-amz-meta-')) { + !field.startsWith('x-amz-meta-') + ) { invalidAttribute = field; } } @@ -68,15 +70,14 @@ function validateSearchParams(searchParams) { // allow using 'replicationStatus' as search param to increase // ease of use, pending metadata search rework // eslint-disable-next-line no-param-reassign - searchParams = searchParams.replace( - 'replication-status', 'replicationInfo.status'); + searchParams = searchParams.replace('replication-status', 'replicationInfo.status'); ast = parser.parse(searchParams); } catch (e) { if (e) { return { - error: errorInstances.InvalidArgument - .customizeDescription('Invalid sql where clause ' + - 'sent as search query'), + error: errorInstances.InvalidArgument.customizeDescription( + 'Invalid sql where clause ' + 'sent as search query', + ), }; } } @@ -84,9 +85,10 @@ function validateSearchParams(searchParams) { const invalidAttribute = _validateTree(ast, possibleAttributes); if (invalidAttribute) { return { - error: errorInstances.InvalidArgument - .customizeDescription('Search param ' + - `contains unknown attribute: ${invalidAttribute}`) }; + error: errorInstances.InvalidArgument.customizeDescription( + 'Search param ' + `contains unknown attribute: ${invalidAttribute}`, + ), + }; } return { ast, diff --git a/lib/api/apiUtils/object/applyZenkoUserMD.js b/lib/api/apiUtils/object/applyZenkoUserMD.js index 928b56b9f3..48e661804d 100644 --- a/lib/api/apiUtils/object/applyZenkoUserMD.js +++ b/lib/api/apiUtils/object/applyZenkoUserMD.js @@ -9,8 +9,7 @@ const _config = require('../../../Config').config; * @return {undefined} */ function applyZenkoUserMD(metaHeaders) { - if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' && - !metaHeaders[zenkoIDHeader]) { + if (process.env.REMOTE_MANAGEMENT_DISABLE === '0' && !metaHeaders[zenkoIDHeader]) { // eslint-disable-next-line no-param-reassign metaHeaders[zenkoIDHeader] = _config.getPublicInstanceId(); } diff --git a/lib/api/apiUtils/object/checkHttpHeadersSize.js b/lib/api/apiUtils/object/checkHttpHeadersSize.js index 01cf136d7d..9d653c25e7 100644 --- a/lib/api/apiUtils/object/checkHttpHeadersSize.js +++ b/lib/api/apiUtils/object/checkHttpHeadersSize.js @@ -10,8 +10,7 @@ function checkHttpHeadersSize(requestHeaders) { let httpHeadersSize = 0; Object.keys(requestHeaders).forEach(header => { - httpHeadersSize += Buffer.byteLength(header, 'utf8') + - Buffer.byteLength(requestHeaders[header], 'utf8'); + httpHeadersSize += Buffer.byteLength(header, 'utf8') + Buffer.byteLength(requestHeaders[header], 'utf8'); }); if (httpHeadersSize > maxHttpHeadersSize) { diff --git a/lib/api/apiUtils/object/checkReadLocation.js b/lib/api/apiUtils/object/checkReadLocation.js index ce21fee04f..13713385de 100644 --- a/lib/api/apiUtils/object/checkReadLocation.js +++ b/lib/api/apiUtils/object/checkReadLocation.js @@ -11,10 +11,8 @@ function checkReadLocation(config, locationName, objectKey, bucketName) { const readLocation = config.getLocationConstraint(locationName); if (readLocation) { - const bucketMatch = readLocation.details && - readLocation.details.bucketMatch; - const backendKey = bucketMatch ? objectKey : - `${bucketName}/${objectKey}`; + const bucketMatch = readLocation.details && readLocation.details.bucketMatch; + const backendKey = bucketMatch ? objectKey : `${bucketName}/${objectKey}`; return { location: locationName, key: backendKey, diff --git a/lib/api/apiUtils/object/checkUserMetadataSize.js b/lib/api/apiUtils/object/checkUserMetadataSize.js index eb8ac11a9c..28e83e55a7 100644 --- a/lib/api/apiUtils/object/checkUserMetadataSize.js +++ b/lib/api/apiUtils/object/checkUserMetadataSize.js @@ -1,5 +1,4 @@ -const { maximumMetaHeadersSize, - invalidObjectUserMetadataHeader } = require('../../../../constants'); +const { maximumMetaHeadersSize, invalidObjectUserMetadataHeader } = require('../../../../constants'); /** * Checks the size of the user metadata in the object metadata and removes @@ -13,8 +12,7 @@ const { maximumMetaHeadersSize, function checkUserMetadataSize(responseMetadata) { let userMetadataSize = 0; // collect the user metadata keys from the object metadata - const userMetadataHeaders = Object.keys(responseMetadata) - .filter(key => key.startsWith('x-amz-meta-')); + const userMetadataHeaders = Object.keys(responseMetadata).filter(key => key.startsWith('x-amz-meta-')); // compute the size of all user metadata key and its value userMetadataHeaders.forEach(header => { userMetadataSize += header.length + responseMetadata[header].length; diff --git a/lib/api/apiUtils/object/corsResponse.js b/lib/api/apiUtils/object/corsResponse.js index 01020b5122..e8cd86df39 100644 --- a/lib/api/apiUtils/object/corsResponse.js +++ b/lib/api/apiUtils/object/corsResponse.js @@ -1,9 +1,9 @@ /** _matchesValue - compare two values to determine if they match -* @param {string} allowedValue - an allowed value in a CORS rule; -* may contain wildcards -* @param {string} value - value from CORS request -* @return {boolean} - true/false -*/ + * @param {string} allowedValue - an allowed value in a CORS rule; + * may contain wildcards + * @param {string} value - value from CORS request + * @return {boolean} - true/false + */ function _matchesValue(allowedValue, value) { const wildcardIndex = allowedValue.indexOf('*'); // If no wildcards, simply return whether strings are equal @@ -14,27 +14,28 @@ function _matchesValue(allowedValue, value) { // and after the wildcard const beginValue = allowedValue.substring(0, wildcardIndex); const endValue = allowedValue.substring(wildcardIndex + 1); - return (value.startsWith(beginValue) && value.endsWith(endValue)); + return value.startsWith(beginValue) && value.endsWith(endValue); } /** _matchesOneOf - check if header matches any AllowedHeaders of a rule -* @param {string[]} allowedHeaders - headers allowed in CORS rule -* @param {string} header - header from CORS request -* @return {boolean} - true/false -*/ + * @param {string[]} allowedHeaders - headers allowed in CORS rule + * @param {string} header - header from CORS request + * @return {boolean} - true/false + */ function _matchesOneOf(allowedHeaders, header) { return allowedHeaders.some(allowedHeader => // AllowedHeaders may have been stored with uppercase letters // during putBucketCors; ignore case when searching for match - _matchesValue(allowedHeader.toLowerCase(), header)); + _matchesValue(allowedHeader.toLowerCase(), header), + ); } /** _headersMatchRule - check if headers match AllowedHeaders of rule -* @param {string[]} headers - the value of the 'Access-Control-Request-Headers' -* in an OPTIONS request -* @param {string[]} allowedHeaders - AllowedHeaders of a CORS rule -* @return {boolean} - true/false -*/ + * @param {string[]} headers - the value of the 'Access-Control-Request-Headers' + * in an OPTIONS request + * @param {string[]} allowedHeaders - AllowedHeaders of a CORS rule + * @return {boolean} - true/false + */ function _headersMatchRule(headers, allowedHeaders) { if (!allowedHeaders) { return false; @@ -46,33 +47,31 @@ function _headersMatchRule(headers, allowedHeaders) { } /** _findCorsRule - Return first matching rule in cors rules that permits -* CORS request -* @param {object[]} rules - array of rules -* @param {string} [rules.id] - optional id to identify rule -* @param {string[]} rules[].allowedMethods - methods allowed for CORS -* @param {string[]} rules[].allowedOrigins - origins allowed for CORS -* @param {string[]} [rules[].allowedHeaders] - headers allowed in an -* OPTIONS request via the Access-Control-Request-Headers header -* @param {number} [rules[].maxAgeSeconds] - seconds browsers should cache -* OPTIONS response -* @param {string[]} [rules[].exposeHeaders] - headers to expose to external -* applications -* @param {string} origin - origin of CORS request -* @param {string} method - Access-Control-Request-Method header value in -* an OPTIONS request and the actual method in any other request -* @param {string[]} [headers] - Access-Control-Request-Headers header value -* in a preflight CORS request -* @return {(null|object)} - matching rule if found; null if no match -*/ + * CORS request + * @param {object[]} rules - array of rules + * @param {string} [rules.id] - optional id to identify rule + * @param {string[]} rules[].allowedMethods - methods allowed for CORS + * @param {string[]} rules[].allowedOrigins - origins allowed for CORS + * @param {string[]} [rules[].allowedHeaders] - headers allowed in an + * OPTIONS request via the Access-Control-Request-Headers header + * @param {number} [rules[].maxAgeSeconds] - seconds browsers should cache + * OPTIONS response + * @param {string[]} [rules[].exposeHeaders] - headers to expose to external + * applications + * @param {string} origin - origin of CORS request + * @param {string} method - Access-Control-Request-Method header value in + * an OPTIONS request and the actual method in any other request + * @param {string[]} [headers] - Access-Control-Request-Headers header value + * in a preflight CORS request + * @return {(null|object)} - matching rule if found; null if no match + */ function findCorsRule(rules, origin, method, headers) { return rules.find(rule => { if (rule.allowedMethods.indexOf(method) === -1) { return false; - } else if (!rule.allowedOrigins.some(allowedOrigin => - _matchesValue(allowedOrigin, origin))) { + } else if (!rule.allowedOrigins.some(allowedOrigin => _matchesValue(allowedOrigin, origin))) { return false; - } else if (headers && - !_headersMatchRule(headers, rule.allowedHeaders)) { + } else if (headers && !_headersMatchRule(headers, rule.allowedHeaders)) { return false; } return true; @@ -80,32 +79,30 @@ function findCorsRule(rules, origin, method, headers) { } /** _gatherResHeaders - Collect headers to return in response -* @param {object} rule - array of rules -* @param {string} [rule.id] - optional id to identify rule -* @param {string[]} rule[].allowedMethods - methods allowed for CORS -* @param {string[]} rule[].allowedOrigins - origins allowed for CORS -* @param {string[]} [rule[].allowedHeaders] - headers allowed in an -* OPTIONS request via the Access-Control-Request-Headers header -* @param {number} [rule[].maxAgeSeconds] - seconds browsers should cache -* OPTIONS response -* @param {string[]} [rule[].exposeHeaders] - headers to expose to external -* applications -* @param {string} origin - origin of CORS request -* @param {string} method - Access-Control-Request-Method header value in -* an OPTIONS request and the actual method in any other request -* @param {string[]} [headers] - Access-Control-Request-Headers header value -* in a preflight CORS request -* @param {boolean} [isPreflight] - indicates if cors headers are being gathered -* for a CORS preflight request -* @return {object} resHeaders - headers to include in response -*/ -function generateCorsResHeaders(rule, origin, method, headers, -isPreflight) { + * @param {object} rule - array of rules + * @param {string} [rule.id] - optional id to identify rule + * @param {string[]} rule[].allowedMethods - methods allowed for CORS + * @param {string[]} rule[].allowedOrigins - origins allowed for CORS + * @param {string[]} [rule[].allowedHeaders] - headers allowed in an + * OPTIONS request via the Access-Control-Request-Headers header + * @param {number} [rule[].maxAgeSeconds] - seconds browsers should cache + * OPTIONS response + * @param {string[]} [rule[].exposeHeaders] - headers to expose to external + * applications + * @param {string} origin - origin of CORS request + * @param {string} method - Access-Control-Request-Method header value in + * an OPTIONS request and the actual method in any other request + * @param {string[]} [headers] - Access-Control-Request-Headers header value + * in a preflight CORS request + * @param {boolean} [isPreflight] - indicates if cors headers are being gathered + * for a CORS preflight request + * @return {object} resHeaders - headers to include in response + */ +function generateCorsResHeaders(rule, origin, method, headers, isPreflight) { const resHeaders = { 'access-control-max-age': rule.maxAgeSeconds, 'access-control-allow-methods': rule.allowedMethods.join(', '), - 'vary': - 'Origin, Access-Control-Request-Headers, Access-Control-Request-Method', + vary: 'Origin, Access-Control-Request-Headers, Access-Control-Request-Method', }; // send back '*' if any origin allowed; otherwise send back // request Origin value @@ -121,8 +118,7 @@ isPreflight) { resHeaders['access-control-allow-headers'] = headers.join(', '); } if (rule.exposeHeaders) { - resHeaders['access-control-expose-headers'] = - rule.exposeHeaders.join(', '); + resHeaders['access-control-expose-headers'] = rule.exposeHeaders.join(', '); } if (isPreflight) { resHeaders['content-length'] = 0; diff --git a/lib/api/apiUtils/object/expirationHeaders.js b/lib/api/apiUtils/object/expirationHeaders.js index 6edcedeb9d..36361f4f95 100644 --- a/lib/api/apiUtils/object/expirationHeaders.js +++ b/lib/api/apiUtils/object/expirationHeaders.js @@ -1,16 +1,8 @@ const { LifecycleConfiguration } = require('arsenal').models; -const { - LifecycleDateTime, - LifecycleUtils, -} = require('arsenal').s3middleware.lifecycleHelpers; +const { LifecycleDateTime, LifecycleUtils } = require('arsenal').s3middleware.lifecycleHelpers; const { config } = require('../../../Config'); -const { - expireOneDayEarlier, - transitionOneDayEarlier, - timeProgressionFactor, - scaledMsPerDay, -} = config.getTimeOptions(); +const { expireOneDayEarlier, transitionOneDayEarlier, timeProgressionFactor, scaledMsPerDay } = config.getTimeOptions(); const lifecycleDateTime = new LifecycleDateTime({ transitionOneDayEarlier, @@ -21,7 +13,7 @@ const lifecycleDateTime = new LifecycleDateTime({ const lifecycleUtils = new LifecycleUtils(config.supportedLifecycleRules, lifecycleDateTime, timeProgressionFactor); function calculateDate(objDate, expDays, datetime) { - return new Date(datetime.getTimestamp(objDate) + (expDays * scaledMsPerDay)); + return new Date(datetime.getTimestamp(objDate) + expDays * scaledMsPerDay); } function formatExpirationHeader(date, id) { @@ -35,13 +27,9 @@ const AMZ_ABORT_DATE_HEADER = 'x-amz-abort-date'; // format: x-amz-abort-rule-id: "rule id" const AMZ_ABORT_ID_HEADER = 'x-amz-abort-rule-id'; - function _generateExpHeadersObjects(rules, params, datetime) { const tags = { - TagSet: params.tags - ? Object.keys(params.tags) - .map(key => ({ Key: key, Value: params.tags[key] })) - : [], + TagSet: params.tags ? Object.keys(params.tags).map(key => ({ Key: key, Value: params.tags[key] })) : [], }; const objectInfo = { Key: params.key }; @@ -80,11 +68,7 @@ function _generateExpHeadresMPU(rules, params, datetime) { if (applicable.AbortIncompleteMultipartUpload) { const rule = applicable.AbortIncompleteMultipartUpload; - const date = calculateDate( - params.date, - rule.DaysAfterInitiation, - datetime - ); + const date = calculateDate(params.date, rule.DaysAfterInitiation, datetime); return { [AMZ_ABORT_ID_HEADER]: encodeURIComponent(rule.ID), diff --git a/lib/api/apiUtils/object/getReplicationBackendDataLocator.js b/lib/api/apiUtils/object/getReplicationBackendDataLocator.js index b5ba4956c8..f04d011ba1 100644 --- a/lib/api/apiUtils/object/getReplicationBackendDataLocator.js +++ b/lib/api/apiUtils/object/getReplicationBackendDataLocator.js @@ -26,25 +26,26 @@ const { errorInstances } = require('arsenal'); */ function getReplicationBackendDataLocator(locationObj, replicationInfo) { const repBackendResult = {}; - const locMatch = replicationInfo.backends.find( - backend => backend.site === locationObj.location); + const locMatch = replicationInfo.backends.find(backend => backend.site === locationObj.location); if (!locMatch) { - repBackendResult.error = errorInstances.InvalidLocationConstraint. - customizeDescription('Object is not replicated to location ' + - 'passed in location header'); + repBackendResult.error = errorInstances.InvalidLocationConstraint.customizeDescription( + 'Object is not replicated to location ' + 'passed in location header', + ); return repBackendResult; } repBackendResult.status = locMatch.status; if (['PENDING', 'FAILED'].includes(locMatch.status)) { - repBackendResult.reason = - `Object replication to specified backend is ${locMatch.status}`; + repBackendResult.reason = `Object replication to specified backend is ${locMatch.status}`; return repBackendResult; } - repBackendResult.dataLocator = [{ - key: locationObj.key, - dataStoreName: locationObj.location, - dataStoreType: locationObj.locationType, - dataStoreVersionId: locMatch.dataStoreVersionId }]; + repBackendResult.dataLocator = [ + { + key: locationObj.key, + dataStoreName: locationObj.location, + dataStoreType: locationObj.locationType, + dataStoreVersionId: locMatch.dataStoreVersionId, + }, + ]; return repBackendResult; } diff --git a/lib/api/apiUtils/object/locationConstraintCheck.js b/lib/api/apiUtils/object/locationConstraintCheck.js index bc87fc249b..cb12ce3531 100644 --- a/lib/api/apiUtils/object/locationConstraintCheck.js +++ b/lib/api/apiUtils/object/locationConstraintCheck.js @@ -20,28 +20,33 @@ function locationConstraintCheck(request, metaHeaders, bucket, log) { let objectLocationConstraint; if (metaHeaders) { - objectLocationConstraint = - metaHeaders[constants.objectLocationConstraintHeader]; + objectLocationConstraint = metaHeaders[constants.objectLocationConstraintHeader]; } else { - objectLocationConstraint = request - .headers[constants.objectLocationConstraintHeader]; + objectLocationConstraint = request.headers[constants.objectLocationConstraintHeader]; } const bucketLocationConstraint = bucket.getLocationConstraint(); const requestEndpoint = request.parsedHost; - const controllingBackend = BackendInfo.controllingBackendParam(config, - objectLocationConstraint, bucketLocationConstraint, - requestEndpoint, log); + const controllingBackend = BackendInfo.controllingBackendParam( + config, + objectLocationConstraint, + bucketLocationConstraint, + requestEndpoint, + log, + ); if (!controllingBackend.isValid) { backendInfoObj = { - err: errorInstances.InvalidArgument.customizeDescription(controllingBackend. - description), + err: errorInstances.InvalidArgument.customizeDescription(controllingBackend.description), }; return backendInfoObj; } - const backendInfo = new BackendInfo(config, objectLocationConstraint, - bucketLocationConstraint, requestEndpoint, - controllingBackend.legacyLocationConstraint); + const backendInfo = new BackendInfo( + config, + objectLocationConstraint, + bucketLocationConstraint, + requestEndpoint, + controllingBackend.legacyLocationConstraint, + ); backendInfoObj = { err: null, controllingLC: backendInfo.getControllingLocationConstraint(), diff --git a/lib/api/apiUtils/object/locationHeaderCheck.js b/lib/api/apiUtils/object/locationHeaderCheck.js index 6fc3ec24f1..d29baa5b57 100644 --- a/lib/api/apiUtils/object/locationHeaderCheck.js +++ b/lib/api/apiUtils/object/locationHeaderCheck.js @@ -19,11 +19,11 @@ function locationHeaderCheck(headers, objectKey, bucketName) { const validLocation = config.locationConstraints[location]; if (!validLocation) { return errorInstances.InvalidLocationConstraint.customizeDescription( - 'Invalid location constraint specified in header'); + 'Invalid location constraint specified in header', + ); } const bucketMatch = validLocation.details.bucketMatch; - const backendKey = bucketMatch ? objectKey : - `${bucketName}/${objectKey}`; + const backendKey = bucketMatch ? objectKey : `${bucketName}/${objectKey}`; return { location, key: backendKey, diff --git a/lib/api/apiUtils/object/locationKeysHaveChanged.js b/lib/api/apiUtils/object/locationKeysHaveChanged.js index 41c8560023..0f372cefdd 100644 --- a/lib/api/apiUtils/object/locationKeysHaveChanged.js +++ b/lib/api/apiUtils/object/locationKeysHaveChanged.js @@ -1,18 +1,18 @@ /** -* Check if all keys that exist in the current list which will be used -* in composing object are not present in the old object's list. -* -* This method can be used to check against accidentally removing data -* keys due to instability from the metadata layer, or for replay -* detection in general. -* -* @param {array|string|null} prev - list of keys from the object being -* overwritten -* @param {array|null} curr - list of keys to be used in composing -* current object -* @returns {boolean} true if no key in `curr` is present in `prev`, -* false otherwise -*/ + * Check if all keys that exist in the current list which will be used + * in composing object are not present in the old object's list. + * + * This method can be used to check against accidentally removing data + * keys due to instability from the metadata layer, or for replay + * detection in general. + * + * @param {array|string|null} prev - list of keys from the object being + * overwritten + * @param {array|null} curr - list of keys to be used in composing + * current object + * @returns {boolean} true if no key in `curr` is present in `prev`, + * false otherwise + */ function locationKeysHaveChanged(prev, curr) { if (!prev || prev.length === 0 || !curr) { return true; diff --git a/lib/api/apiUtils/object/locationStorageCheck.js b/lib/api/apiUtils/object/locationStorageCheck.js index 88cbbb18a3..6129902377 100644 --- a/lib/api/apiUtils/object/locationStorageCheck.js +++ b/lib/api/apiUtils/object/locationStorageCheck.js @@ -1,8 +1,7 @@ const { errorInstances } = require('arsenal'); const { config } = require('../../../Config'); -const { getLocationMetric, pushLocationMetric } = - require('../../../utapi/utilities'); +const { getLocationMetric, pushLocationMetric } = require('../../../utapi/utilities'); function _gbToBytes(gb) { return gb * 1024 * 1024 * 1024; @@ -37,9 +36,11 @@ function locationStorageCheck(location, updateSize, log, cb) { const newStorageSize = parseInt(bytesStored, 10) + updateSize; const sizeLimitBytes = _gbToBytes(sizeLimitGB); if (sizeLimitBytes < newStorageSize) { - return cb(errorInstances.AccessDenied.customizeDescription( - `The assigned storage space limit for location ${location} ` + - 'will be exceeded')); + return cb( + errorInstances.AccessDenied.customizeDescription( + `The assigned storage space limit for location ${location} ` + 'will be exceeded', + ), + ); } return pushLocationMetric(location, updateSize, log, cb); }); diff --git a/lib/api/apiUtils/object/objectLockHelpers.js b/lib/api/apiUtils/object/objectLockHelpers.js index e2e18fc640..ede44835dc 100644 --- a/lib/api/apiUtils/object/objectLockHelpers.js +++ b/lib/api/apiUtils/object/objectLockHelpers.js @@ -22,8 +22,7 @@ function calculateRetainUntilDate(retention) { // Calculate the number of days to retain the lock on the object const retainUntilDays = days || years * 365; const retainUntilDaysInMs = retainUntilDays * scaledMsPerDay; - const retainUntilDate - = date.add(retainUntilDaysInMs, 'ms'); + const retainUntilDate = date.add(retainUntilDaysInMs, 'ms'); return retainUntilDate.toISOString(); } /** @@ -40,33 +39,26 @@ function validateHeaders(bucket, headers, log) { const objectLockMode = headers['x-amz-object-lock-mode']; // If retention headers or legal hold header present but // object lock is not enabled on the bucket return error - if ((objectLockDate || objectLockMode || objectLegalHold) - && !bucketObjectLockEnabled) { + if ((objectLockDate || objectLockMode || objectLegalHold) && !bucketObjectLockEnabled) { log.trace('bucket is missing ObjectLockConfiguration'); - return errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing ObjectLockConfiguration'); + return errorInstances.InvalidRequest.customizeDescription('Bucket is missing ObjectLockConfiguration'); } - if ((objectLockMode || objectLockDate) && - !(objectLockMode && objectLockDate)) { + if ((objectLockMode || objectLockDate) && !(objectLockMode && objectLockDate)) { return errorInstances.InvalidArgument.customizeDescription( - 'x-amz-object-lock-retain-until-date and ' + - 'x-amz-object-lock-mode must both be supplied', + 'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-mode must both be supplied', ); } const validModes = new Set(['GOVERNANCE', 'COMPLIANCE']); if (objectLockMode && !validModes.has(objectLockMode)) { - return errorInstances.InvalidArgument.customizeDescription( - 'Unknown wormMode directive'); + return errorInstances.InvalidArgument.customizeDescription('Unknown wormMode directive'); } const validLegalHolds = new Set(['ON', 'OFF']); if (objectLegalHold && !validLegalHolds.has(objectLegalHold)) { - return errorInstances.InvalidArgument.customizeDescription( - 'Legal hold status must be one of "ON", "OFF"'); + return errorInstances.InvalidArgument.customizeDescription('Legal hold status must be one of "ON", "OFF"'); } const currentDate = new Date().toISOString(); if (objectLockMode && objectLockDate <= currentDate) { - return errorInstances.InvalidArgument.customizeDescription( - 'The retain until date must be in the future!'); + return errorInstances.InvalidArgument.customizeDescription('The retain until date must be in the future!'); } return null; } @@ -121,8 +113,7 @@ function compareObjectLockInformation(headers, defaultRetention) { function setObjectLockInformation(headers, md, defaultRetention) { // Stores retention information if object either has its own retention // configuration or default retention configuration from its bucket - const finalObjectLockInfo = - compareObjectLockInformation(headers, defaultRetention); + const finalObjectLockInfo = compareObjectLockInformation(headers, defaultRetention); if (finalObjectLockInfo.retentionInfo) { md.setRetentionMode(finalObjectLockInfo.retentionInfo.mode); md.setRetentionDate(finalObjectLockInfo.retentionInfo.date); @@ -261,7 +252,6 @@ function hasGovernanceBypassHeader(headers) { return bypassHeader.toLowerCase() === 'true'; } - /** * checkUserGovernanceBypass * @@ -276,10 +266,9 @@ function hasGovernanceBypassHeader(headers) { * @returns {undefined} - */ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, cb) { - log.trace( - 'object in GOVERNANCE mode and is user, checking for attached policies', - { method: 'checkUserPolicyGovernanceBypass' }, - ); + log.trace('object in GOVERNANCE mode and is user, checking for attached policies', { + method: 'checkUserPolicyGovernanceBypass', + }); const authParams = auth.server.extractParams(request, log, 's3', request.query); const ip = policies.requestUtils.getClientIp(request, config); @@ -301,41 +290,41 @@ function checkUserGovernanceBypass(request, authInfo, bucketMD, objectKey, log, signatureAge: authParams.params.data.signatureAge, }, }; - return vault.checkPolicies(requestContextParams, - authInfo.getArn(), log, (err, authorizationResults) => { - if (err) { - return cb(err); - } - const explicitDenyExists = authorizationResults.some( - authzResult => authzResult.isAllowed === false && !authzResult.isImplicit); - if (explicitDenyExists) { - log.trace('authorization check failed for user', - { - 'method': 'checkUserPolicyGovernanceBypass', - 's3:BypassGovernanceRetention': false, - }); - return cb(errors.AccessDenied); - } - // Convert authorization results into an easier to handle format - const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => { - const apiMethod = authorizationResults[idx].action; - // eslint-disable-next-line no-param-reassign - acc[apiMethod] = curr.isImplicit; - return acc; - }, {}); + return vault.checkPolicies(requestContextParams, authInfo.getArn(), log, (err, authorizationResults) => { + if (err) { + return cb(err); + } + const explicitDenyExists = authorizationResults.some( + authzResult => authzResult.isAllowed === false && !authzResult.isImplicit, + ); + if (explicitDenyExists) { + log.trace('authorization check failed for user', { + method: 'checkUserPolicyGovernanceBypass', + 's3:BypassGovernanceRetention': false, + }); + return cb(errors.AccessDenied); + } + // Convert authorization results into an easier to handle format + const actionImplicitDenies = authorizationResults.reduce((acc, curr, idx) => { + const apiMethod = authorizationResults[idx].action; + // eslint-disable-next-line no-param-reassign + acc[apiMethod] = curr.isImplicit; + return acc; + }, {}); - // Evaluate against the bucket policies - const areAllActionsAllowed = evaluateBucketPolicyWithIAM( - bucketMD, - Object.keys(actionImplicitDenies), - authInfo.getCanonicalID(), - authInfo, - actionImplicitDenies, - log, - request); + // Evaluate against the bucket policies + const areAllActionsAllowed = evaluateBucketPolicyWithIAM( + bucketMD, + Object.keys(actionImplicitDenies), + authInfo.getCanonicalID(), + authInfo, + actionImplicitDenies, + log, + request, + ); - return cb(areAllActionsAllowed === true ? null : errors.AccessDenied); - }); + return cb(areAllActionsAllowed === true ? null : errors.AccessDenied); + }); } module.exports = { diff --git a/lib/api/apiUtils/object/objectRestore.js b/lib/api/apiUtils/object/objectRestore.js index 7912416258..bfa3ce0301 100644 --- a/lib/api/apiUtils/object/objectRestore.js +++ b/lib/api/apiUtils/object/objectRestore.js @@ -42,12 +42,11 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { const decodedVidResult = decodeVersionId(request.query); if (decodedVidResult instanceof Error) { - log.trace('invalid versionId query', - { - method: METHOD, - versionId: request.query.versionId, - error: decodedVidResult, - }); + log.trace('invalid versionId query', { + method: METHOD, + versionId: request.query.versionId, + error: decodedVidResult, + }); return process.nextTick(() => callback(decodedVidResult)); } @@ -69,36 +68,41 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { request, }; - return async.waterfall([ + return async.waterfall( + [ // get metadata of bucket and object function validateBucketAndObject(next) { - return mdUtils.standardMetadataValidateBucketAndObj(mdValueParams, request.actionImplicitDenies, - log, (err, bucketMD, objectMD) => { - if (err) { - log.trace('request authorization failed', { method: METHOD, error: err }); - return next(err); - } - // Call back error if object metadata could not be obtained - if (!objectMD) { - const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey; - log.trace('error no object metadata found', { method: METHOD, error: err }); - return next(err, bucketMD); - } - // If object metadata is delete marker, - // call back NoSuchKey or MethodNotAllowed depending on specifying versionId - if (objectMD.isDeleteMarker) { - let err = errorInstances.NoSuchKey; - if (decodedVidResult) { - err = errorInstances.MethodNotAllowed; + return mdUtils.standardMetadataValidateBucketAndObj( + mdValueParams, + request.actionImplicitDenies, + log, + (err, bucketMD, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: METHOD, error: err }); + return next(err); } - log.trace('version is a delete marker', { method: METHOD, error: err }); - return next(err, bucketMD, objectMD); - } - log.debug('acquired the object metadata.', { - 'method': METHOD, - }); - return next(null, bucketMD, objectMD); - }); + // Call back error if object metadata could not be obtained + if (!objectMD) { + const err = decodedVidResult ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: METHOD, error: err }); + return next(err, bucketMD); + } + // If object metadata is delete marker, + // call back NoSuchKey or MethodNotAllowed depending on specifying versionId + if (objectMD.isDeleteMarker) { + let err = errorInstances.NoSuchKey; + if (decodedVidResult) { + err = errorInstances.MethodNotAllowed; + } + log.trace('version is a delete marker', { method: METHOD, error: err }); + return next(err, bucketMD, objectMD); + } + log.debug('acquired the object metadata.', { + method: METHOD, + }); + return next(null, bucketMD, objectMD); + }, + ); }, // generate restore param obj from xml of request body and check tier validity @@ -118,39 +122,47 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { }, // start restore process function startRestore(bucketMD, objectMD, restoreInfo, next) { - return coldStorage.startRestore(objectMD, restoreInfo, log, - (err, _isObjectRestored) => { - isObjectRestored = _isObjectRestored; - return next(err, bucketMD, objectMD); - }); + return coldStorage.startRestore(objectMD, restoreInfo, log, (err, _isObjectRestored) => { + isObjectRestored = _isObjectRestored; + return next(err, bucketMD, objectMD); + }); }, function evaluateQuotas(bucketMD, objectMD, next) { if (isObjectRestored) { return next(null, bucketMD, objectMD); } - const actions = Array.isArray(mdValueParams.requestType) ? - mdValueParams.requestType : [mdValueParams.requestType]; + const actions = Array.isArray(mdValueParams.requestType) + ? mdValueParams.requestType + : [mdValueParams.requestType]; const bytes = processBytesToWrite(request.apiMethod, bucketMD, mdValueParams.versionId, 0, objectMD); - return validateQuotas(request, bucketMD, request.accountQuotas, actions, request.apiMethod, bytes, - false, log, err => next(err, bucketMD, objectMD)); + return validateQuotas( + request, + bucketMD, + request.accountQuotas, + actions, + request.apiMethod, + bytes, + false, + log, + err => next(err, bucketMD, objectMD), + ); }, function updateObjectMD(bucketMD, objectMD, next) { const params = objectMD.versionId ? { versionId: objectMD.versionId } : {}; - metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params, - log, err => next(err, bucketMD, objectMD)); + metadata.putObjectMD(bucketMD.getName(), objectKey, objectMD, params, log, err => + next(err, bucketMD, objectMD), + ); }, ], (err, bucketMD) => { // generate CORS response header const responseHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); if (err) { - log.trace('error processing request', - { - method: METHOD, - error: err, - }); - monitoring.promMetrics( - 'POST', bucketName, err.code, 'restoreObject'); + log.trace('error processing request', { + method: METHOD, + error: err, + }); + monitoring.promMetrics('POST', bucketName, err.code, 'restoreObject'); return callback(err, err.code, responseHeaders); } pushMetric('restoreObject', log, { @@ -158,15 +170,13 @@ function objectRestore(metadata, mdUtils, userInfo, request, log, callback) { bucket: bucketName, }); if (isObjectRestored) { - monitoring.promMetrics( - 'POST', bucketName, '200', 'restoreObject'); + monitoring.promMetrics('POST', bucketName, '200', 'restoreObject'); return callback(null, 200, responseHeaders); } - monitoring.promMetrics( - 'POST', bucketName, '202', 'restoreObject'); + monitoring.promMetrics('POST', bucketName, '202', 'restoreObject'); return callback(null, 202, responseHeaders); - }); + }, + ); } - module.exports = objectRestore; diff --git a/lib/api/apiUtils/object/parseCopySource.js b/lib/api/apiUtils/object/parseCopySource.js index a28770ded3..27a262dbc0 100644 --- a/lib/api/apiUtils/object/parseCopySource.js +++ b/lib/api/apiUtils/object/parseCopySource.js @@ -26,8 +26,7 @@ function parseCopySource(apiMethod, copySourceHeader) { // Pull the source bucket and source object separated by / const sourceBucket = source.slice(0, slashSeparator); const sourceObject = source.slice(slashSeparator + 1); - const sourceVersionId = - decodeVersionId(query ? querystring.parse(query) : undefined); + const sourceVersionId = decodeVersionId(query ? querystring.parse(query) : undefined); if (sourceVersionId instanceof Error) { const err = sourceVersionId; return { parsingError: err }; diff --git a/lib/api/apiUtils/object/partInfo.js b/lib/api/apiUtils/object/partInfo.js index c8715d3628..3480bd0090 100644 --- a/lib/api/apiUtils/object/partInfo.js +++ b/lib/api/apiUtils/object/partInfo.js @@ -6,8 +6,7 @@ */ function getPartNumber(query) { if (query && query.partNumber !== undefined) { - return Number.isNaN(query.partNumber) ? - 0 : Number.parseInt(query.partNumber, 10); + return Number.isNaN(query.partNumber) ? 0 : Number.parseInt(query.partNumber, 10); } return undefined; } @@ -21,14 +20,12 @@ function getPartNumber(query) { function getPartSize(objMD, partNumber) { let size; let locationPartNumber; - if (partNumber && objMD && objMD.location - && objMD.location.length >= partNumber) { + if (partNumber && objMD && objMD.location && objMD.location.length >= partNumber) { const locations = []; for (let i = 0; i < objMD.location.length; i++) { const { dataStoreETag } = objMD.location[i]; if (dataStoreETag) { - locationPartNumber = - Number.parseInt(dataStoreETag.split(':')[0], 10); + locationPartNumber = Number.parseInt(dataStoreETag.split(':')[0], 10); } else { /** * Location objects prior to GA7.1 do not include the diff --git a/lib/api/apiUtils/object/setPartRanges.js b/lib/api/apiUtils/object/setPartRanges.js index 9a2587d1cf..f36a1273c6 100644 --- a/lib/api/apiUtils/object/setPartRanges.js +++ b/lib/api/apiUtils/object/setPartRanges.js @@ -28,8 +28,7 @@ function setPartRanges(dataLocations, outerRange) { // be allowed, so not an issue that size not modified here. if (dataLocations[0].size) { const partSize = parseInt(dataLocations[0].size, 10); - soleLocation.size = - Math.min(partSize, end - begin + 1).toString(); + soleLocation.size = Math.min(partSize, end - begin + 1).toString(); } parsedLocations.push(soleLocation); return parsedLocations; @@ -73,8 +72,7 @@ function setPartRanges(dataLocations, outerRange) { // Use full remaining part if remaining partSize is less // than byte range we need to satisfy. Or use byte range // we need to satisfy taking into account any startOffset - const endPart = Math.min(partSize - 1, - max - total + startOffset - 1); + const endPart = Math.min(partSize - 1, max - total + startOffset - 1); partWithRange.range = [startOffset, endPart]; // modify size to be stored for object put part copy partWithRange.size = (endPart - startOffset + 1).toString(); diff --git a/lib/api/apiUtils/object/setUpCopyLocator.js b/lib/api/apiUtils/object/setUpCopyLocator.js index a232fa3620..5b6cef17e7 100644 --- a/lib/api/apiUtils/object/setUpCopyLocator.js +++ b/lib/api/apiUtils/object/setUpCopyLocator.js @@ -1,8 +1,5 @@ const { errors, errorInstances } = require('arsenal'); -const { - parseRangeSpec, - parseRange, -} = require('arsenal').network.http.utils; +const { parseRangeSpec, parseRange } = require('arsenal').network.http.utils; const constants = require('../../../../constants'); const setPartRanges = require('./setPartRanges'); @@ -15,7 +12,8 @@ const setPartRanges = require('./setPartRanges'); function parseRangeHeader(header) { const { error } = parseRangeSpec(header); if (error) { - const description = 'The x-amz-copy-source-range value must be ' + + const description = + 'The x-amz-copy-source-range value must be ' + 'of the form bytes=first-last where first and last are the ' + 'zero-based offsets of the first and last bytes to copy'; return error.customizeDescription(description); @@ -42,21 +40,17 @@ function setUpCopyLocator(sourceObjMD, rangeHeader, log) { // To provide for backwards compatibility before // md-model-version 2, need to handle cases where // objMD.location is just a string - dataLocator = Array.isArray(sourceObjMD.location) ? - sourceObjMD.location : [{ key: sourceObjMD.location }]; + dataLocator = Array.isArray(sourceObjMD.location) ? sourceObjMD.location : [{ key: sourceObjMD.location }]; } if (sourceObjMD['x-amz-server-side-encryption']) { for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - sourceObjMD['x-amz-server-side-encryption-aws-kms-key-id']; - dataLocator[i].algorithm = - sourceObjMD['x-amz-server-side-encryption']; + dataLocator[i].masterKeyId = sourceObjMD['x-amz-server-side-encryption-aws-kms-key-id']; + dataLocator[i].algorithm = sourceObjMD['x-amz-server-side-encryption']; } } - const sourceSize = - parseInt(sourceObjMD['content-length'], 10); + const sourceSize = parseInt(sourceObjMD['content-length'], 10); let copyObjectSize = sourceSize; if (rangeHeader) { const rangeHeaderError = parseRangeHeader(rangeHeader); @@ -70,15 +64,18 @@ function setUpCopyLocator(sourceObjMD, rangeHeader, log) { // If have a data model before version 2, cannot // support get range copy (do not have size // stored with data locations) - if ((range && dataLocator.length >= 1) && - (dataLocator[0].start === undefined - || dataLocator[0].size === undefined)) { - log.trace('data model before version 2 so ' + - 'cannot support get range copy part'); - return { error: errorInstances.NotImplemented - .customizeDescription('Stored object ' + - 'has legacy data storage model so does' + - ' not support range headers on copy part'), + if ( + range && + dataLocator.length >= 1 && + (dataLocator[0].start === undefined || dataLocator[0].size === undefined) + ) { + log.trace('data model before version 2 so ' + 'cannot support get range copy part'); + return { + error: errorInstances.NotImplemented.customizeDescription( + 'Stored object ' + + 'has legacy data storage model so does' + + ' not support range headers on copy part', + ), }; } if (range) { @@ -87,8 +84,7 @@ function setUpCopyLocator(sourceObjMD, rangeHeader, log) { } } if (copyObjectSize > constants.maximumAllowedPartSize) { - log.debug('copy part size too large', { sourceSize, rangeHeader, - copyObjectSize }); + log.debug('copy part size too large', { sourceSize, rangeHeader, copyObjectSize }); return { error: errors.EntityTooLarge }; } return { dataLocator, copyObjectSize }; diff --git a/lib/api/apiUtils/object/sseHeaders.js b/lib/api/apiUtils/object/sseHeaders.js index 8ed85a828e..6fdc60aee0 100644 --- a/lib/api/apiUtils/object/sseHeaders.js +++ b/lib/api/apiUtils/object/sseHeaders.js @@ -7,8 +7,9 @@ function setSSEHeaders(headers, algo, kmsKey) { headers['x-amz-server-side-encryption'] = algo; if (kmsKey && algo === 'aws:kms') { // eslint-disable-next-line no-param-reassign - headers['x-amz-server-side-encryption-aws-kms-key-id'] = - config.kmsHideScalityArn ? getKeyIdFromArn(kmsKey) : kmsKey; + headers['x-amz-server-side-encryption-aws-kms-key-id'] = config.kmsHideScalityArn + ? getKeyIdFromArn(kmsKey) + : kmsKey; } } } diff --git a/lib/api/apiUtils/object/websiteServing.js b/lib/api/apiUtils/object/websiteServing.js index d5f23f15ba..cf5a25f842 100644 --- a/lib/api/apiUtils/object/websiteServing.js +++ b/lib/api/apiUtils/object/websiteServing.js @@ -22,10 +22,8 @@ function findRoutingRule(routingRules, key, errCode) { // no error condition, will have match on first rule even if later // there is more specific rule with error condition. for (let i = 0; i < routingRules.length; i++) { - const prefixFromRule = - routingRules[i].getCondition().keyPrefixEquals; - const errorCodeFromRule = - routingRules[i].getCondition().httpErrorCodeReturnedEquals; + const prefixFromRule = routingRules[i].getCondition().keyPrefixEquals; + const errorCodeFromRule = routingRules[i].getCondition().httpErrorCodeReturnedEquals; if (prefixFromRule !== undefined) { if (!key.startsWith(prefixFromRule)) { // no key match, move on @@ -34,8 +32,7 @@ function findRoutingRule(routingRules, key, errCode) { // add the prefixFromRule to the redirect info // so we can replaceKeyPrefixWith if that is part of redirect // rule - const redirectInfo = Object.assign({ prefixFromRule }, - routingRules[i].getRedirect()); + const redirectInfo = Object.assign({ prefixFromRule }, routingRules[i].getRedirect()); // have key match so check error code match if (errorCodeFromRule !== undefined) { if (errCode === errorCodeFromRule) { @@ -51,8 +48,7 @@ function findRoutingRule(routingRules, key, errCode) { // we have an error code condition but no key condition if (errorCodeFromRule !== undefined) { if (errCode === errorCodeFromRule) { - const redirectInfo = Object.assign({}, - routingRules[i].getRedirect()); + const redirectInfo = Object.assign({}, routingRules[i].getRedirect()); return redirectInfo; } continue; @@ -97,8 +93,7 @@ function extractRedirectInfo(location) { * @return {boolean} true if valid, false if not */ function validateWebsiteHeader(header) { - return (!header || header.startsWith('/') || - header.startsWith('http://') || header.startsWith('https://')); + return !header || header.startsWith('/') || header.startsWith('http://') || header.startsWith('https://'); } /** @@ -115,10 +110,10 @@ function appendWebsiteIndexDocument(request, indexDocumentSuffix, force = false) // find index document if "directory" sent in request if (reqObjectKey.endsWith('/')) { request.objectKey += indexDocumentSuffix; - // find index document if no key provided + // find index document if no key provided } else if (reqObjectKey === '') { request.objectKey = indexDocumentSuffix; - // force for redirect 302 on folder without trailing / that has an index + // force for redirect 302 on folder without trailing / that has an index } else if (force) { request.objectKey += `/${indexDocumentSuffix}`; } diff --git a/lib/api/apiUtils/rateLimit/cleanup.js b/lib/api/apiUtils/rateLimit/cleanup.js index 3a54fbcfaa..081c047c5b 100644 --- a/lib/api/apiUtils/rateLimit/cleanup.js +++ b/lib/api/apiUtils/rateLimit/cleanup.js @@ -16,7 +16,6 @@ function cleanupJob(log, options = {}) { } } - /** * Start periodic cleanup of expired rate limit counters and cached configs * diff --git a/lib/api/apiUtils/rateLimit/client.js b/lib/api/apiUtils/rateLimit/client.js index 203643fc0f..a41b3afbf0 100644 --- a/lib/api/apiUtils/rateLimit/client.js +++ b/lib/api/apiUtils/rateLimit/client.js @@ -50,22 +50,15 @@ class RateLimitClient { const key = `ratelimit:${resourceClass}:${resourceId}:${measure}:emptyAt`; const now = Date.now(); - this.redis.grantTokens( - key, - requested, - interval, - burstCapacity, - now, - (err, result) => { - if (err) { - return cb(err); - } - - // Result is number of tokens granted (0 if denied, partial if limited) - const granted = parseInt(result, 10); - return cb(null, granted); + this.redis.grantTokens(key, requested, interval, burstCapacity, now, (err, result) => { + if (err) { + return cb(err); } - ); + + // Result is number of tokens granted (0 if denied, partial if limited) + const granted = parseInt(result, 10); + return cb(null, granted); + }); } /** @@ -84,5 +77,5 @@ if (config.rateLimiting.enabled) { module.exports = { instance, - RateLimitClient + RateLimitClient, }; diff --git a/lib/api/backbeat/listLifecycleCurrents.js b/lib/api/backbeat/listLifecycleCurrents.js index 4799418847..3dd723c868 100644 --- a/lib/api/backbeat/listLifecycleCurrents.js +++ b/lib/api/backbeat/listLifecycleCurrents.js @@ -4,12 +4,14 @@ const services = require('../../services'); const { standardMetadataValidateBucket } = require('../../metadata/metadataUtils'); const { pushMetric } = require('../../utapi/utilities'); const monitoring = require('../../utilities/monitoringHandler'); -const { getLocationConstraintErrorMessage, processCurrents, - validateMaxScannedEntries } = require('../apiUtils/object/lifecycle'); +const { + getLocationConstraintErrorMessage, + processCurrents, + validateMaxScannedEntries, +} = require('../apiUtils/object/lifecycle'); const { config } = require('../../Config'); -function handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, isBucketVersioned, log, callback) { +function handleResult(listParams, requestMaxKeys, authInfo, bucketName, list, isBucketVersioned, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; const res = processCurrents(bucketName, listParams, isBucketVersioned, list); @@ -35,18 +37,19 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call const bucketName = request.bucketName; log.debug('processing request', { method: 'listLifecycleCurrents' }); - const requestMaxKeys = params['max-keys'] ? - Number.parseInt(params['max-keys'], 10) : 1000; + const requestMaxKeys = params['max-keys'] ? Number.parseInt(params['max-keys'], 10) : 1000; if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'listLifecycleCurrents'); + monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents'); return callback(errors.InvalidArgument); } const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys); const minEntriesToBeScanned = 1; - const { isValid, maxScannedLifecycleListingEntries } = - validateMaxScannedEntries(params, config, minEntriesToBeScanned); + const { isValid, maxScannedLifecycleListingEntries } = validateMaxScannedEntries( + params, + config, + minEntriesToBeScanned, + ); if (!isValid) { monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleCurrents'); return callback(errors.InvalidArgument); @@ -80,8 +83,7 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { if (err) { log.debug('error processing request', { method: 'metadataValidateBucket', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleCurrents'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleCurrents'); return callback(err, null); } @@ -93,21 +95,35 @@ function listLifecycleCurrents(authInfo, locationConstraints, request, log, call Contents: [], IsTruncated: false, }; - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, emptyList, isBucketVersioned, log, callback); + return handleResult( + listParams, + requestMaxKeys, + authInfo, + bucketName, + emptyList, + isBucketVersioned, + log, + callback, + ); } - return services.getLifecycleListing(bucketName, listParams, log, - (err, list) => { + return services.getLifecycleListing(bucketName, listParams, log, (err, list) => { if (err) { log.debug('error processing request', { method: 'services.getLifecycleListing', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleCurrents'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleCurrents'); return callback(err, null); } - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, isBucketVersioned, log, callback); + return handleResult( + listParams, + requestMaxKeys, + authInfo, + bucketName, + list, + isBucketVersioned, + log, + callback, + ); }); }); } diff --git a/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js b/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js index 89cab46088..48762ae3b3 100644 --- a/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js +++ b/lib/api/backbeat/listLifecycleOrphanDeleteMarkers.js @@ -7,8 +7,7 @@ const monitoring = require('../../utilities/monitoringHandler'); const { processOrphans, validateMaxScannedEntries } = require('../apiUtils/object/lifecycle'); const { config } = require('../../Config'); -function handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, log, callback) { +function handleResult(listParams, requestMaxKeys, authInfo, bucketName, list, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; const res = processOrphans(bucketName, listParams, list); @@ -34,11 +33,9 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request const bucketName = request.bucketName; log.debug('processing request', { method: 'listLifecycleOrphanDeleteMarkers' }); - const requestMaxKeys = params['max-keys'] ? - Number.parseInt(params['max-keys'], 10) : 1000; + const requestMaxKeys = params['max-keys'] ? Number.parseInt(params['max-keys'], 10) : 1000; if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers'); + monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers'); return callback(errors.InvalidArgument); } const actualMaxKeys = Math.min(constants.listingHardLimit, requestMaxKeys); @@ -46,8 +43,11 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request // 3 is required as a minimum because we must scan at least three entries to determine version eligibility. // Two entries representing the master key and the following one representing the non-current version. const minEntriesToBeScanned = 3; - const { isValid, maxScannedLifecycleListingEntries } = - validateMaxScannedEntries(params, config, minEntriesToBeScanned); + const { isValid, maxScannedLifecycleListingEntries } = validateMaxScannedEntries( + params, + config, + minEntriesToBeScanned, + ); if (!isValid) { monitoring.promMetrics('GET', bucketName, 400, 'listLifecycleOrphanDeleteMarkers'); return callback(errors.InvalidArgument); @@ -71,8 +71,7 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { if (err) { log.debug('error processing request', { method: 'metadataValidateBucket', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); return callback(err, null); } @@ -80,8 +79,7 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request const isBucketVersioned = vcfg && (vcfg.Status === 'Enabled' || vcfg.Status === 'Suspended'); if (!isBucketVersioned) { log.debug('bucket is not versioned or suspended'); - return callback(errorInstances.InvalidRequest.customizeDescription( - 'bucket is not versioned'), null); + return callback(errorInstances.InvalidRequest.customizeDescription('bucket is not versioned'), null); } if (!requestMaxKeys) { @@ -89,20 +87,16 @@ function listLifecycleOrphanDeleteMarkers(authInfo, locationConstraints, request Contents: [], IsTruncated: false, }; - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, emptyList, log, callback); + return handleResult(listParams, requestMaxKeys, authInfo, bucketName, emptyList, log, callback); } - return services.getLifecycleListing(bucketName, listParams, log, - (err, list) => { + return services.getLifecycleListing(bucketName, listParams, log, (err, list) => { if (err) { log.debug('error processing request', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); + monitoring.promMetrics('GET', bucketName, err.code, 'listLifecycleOrphanDeleteMarkers'); return callback(err, null); } - return handleResult(listParams, requestMaxKeys, authInfo, - bucketName, list, log, callback); + return handleResult(listParams, requestMaxKeys, authInfo, bucketName, list, log, callback); }); }); } diff --git a/lib/api/bucketDelete.js b/lib/api/bucketDelete.js index 78af94dec4..4f08704692 100644 --- a/lib/api/bucketDelete.js +++ b/lib/api/bucketDelete.js @@ -21,8 +21,7 @@ function bucketDelete(authInfo, request, log, cb) { if (authInfo.isRequesterPublicUser()) { log.debug('operation not available for public user'); - monitoring.promMetrics( - 'DELETE', request.bucketName, 403, 'deleteBucket'); + monitoring.promMetrics('DELETE', request.bucketName, 403, 'deleteBucket'); return cb(errors.AccessDenied); } const bucketName = request.bucketName; @@ -34,35 +33,27 @@ function bucketDelete(authInfo, request, log, cb) { request, }; - return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucketMD) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucketMD); + return standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucketMD) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); + if (err) { + log.debug('error processing request', { method: 'metadataValidateBucket', error: err }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucket'); + return cb(err, corsHeaders); + } + log.trace('passed checks', { method: 'metadataValidateBucket' }); + return deleteBucket(authInfo, bucketMD, bucketName, authInfo.getCanonicalID(), request, log, err => { if (err) { - log.debug('error processing request', - { method: 'metadataValidateBucket', error: err }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucket'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucket'); return cb(err, corsHeaders); } - log.trace('passed checks', - { method: 'metadataValidateBucket' }); - return deleteBucket(authInfo, bucketMD, bucketName, - authInfo.getCanonicalID(), request, log, err => { - if (err) { - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucket'); - return cb(err, corsHeaders); - } - pushMetric('deleteBucket', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '204', 'deleteBucket'); - return cb(null, corsHeaders); - }); + pushMetric('deleteBucket', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('DELETE', bucketName, '204', 'deleteBucket'); + return cb(null, corsHeaders); }); + }); } module.exports = bucketDelete; diff --git a/lib/api/bucketDeleteEncryption.js b/lib/api/bucketDeleteEncryption.js index 35bf57c6c9..9e3a3a6d2f 100644 --- a/lib/api/bucketDeleteEncryption.js +++ b/lib/api/bucketDeleteEncryption.js @@ -25,53 +25,55 @@ function bucketDeleteEncryption(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), - (bucket, next) => { - const sseConfig = bucket.getServerSideEncryption(); + return async.waterfall( + [ + next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), + (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), + (bucket, next) => { + const sseConfig = bucket.getServerSideEncryption(); - if (sseConfig === null) { - return next(null, bucket); - } + if (sseConfig === null) { + return next(null, bucket); + } - const { isAccountEncryptionEnabled, masterKeyId, algorithm, cryptoScheme } = sseConfig; + const { isAccountEncryptionEnabled, masterKeyId, algorithm, cryptoScheme } = sseConfig; - let updatedSseConfig = null; + let updatedSseConfig = null; - if (!isAccountEncryptionEnabled && masterKeyId) { - // Keep the encryption configuration as a "cache" to avoid generating a new master key: - // - if the default encryption master key is defined at the bucket level (!isAccountEncryptionEnabled), - // - and if a bucket-level default encryption key is already set. - // This "cache" is implemented by storing the configuration in the bucket metadata - // with mandatory set to false, making sure it remains hidden for `getBucketEncryption` operations. - // There is no need to cache the configuration if the default encryption master key is - // managed at the account level, as the master key id in that case is stored directly in - // the account metadata. - updatedSseConfig = { - mandatory: false, - algorithm, - cryptoScheme, - masterKeyId, - }; - } + if (!isAccountEncryptionEnabled && masterKeyId) { + // Keep the encryption configuration as a "cache" to avoid generating a new master key: + // - if the default encryption master key is defined at the bucket level (!isAccountEncryptionEnabled), + // - and if a bucket-level default encryption key is already set. + // This "cache" is implemented by storing the configuration in the bucket metadata + // with mandatory set to false, making sure it remains hidden for `getBucketEncryption` operations. + // There is no need to cache the configuration if the default encryption master key is + // managed at the account level, as the master key id in that case is stored directly in + // the account metadata. + updatedSseConfig = { + mandatory: false, + algorithm, + cryptoScheme, + masterKeyId, + }; + } - bucket.setServerSideEncryption(updatedSseConfig); - return metadata.updateBucket(bucketName, bucket, log, err => next(err, bucket)); + bucket.setServerSideEncryption(updatedSseConfig); + return metadata.updateBucket(bucketName, bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketDeleteEncryption' }); + return callback(err, corsHeaders); + } + pushMetric('deleteBucketEncryption', log, { + authInfo, + bucket: bucketName, + }); + return callback(null, corsHeaders); }, - ], - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, method: 'bucketDeleteEncryption' }); - return callback(err, corsHeaders); - } - pushMetric('deleteBucketEncryption', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketDeleteEncryption; diff --git a/lib/api/bucketDeleteLifecycle.js b/lib/api/bucketDeleteLifecycle.js index 31a4148a74..db1b123454 100644 --- a/lib/api/bucketDeleteLifecycle.js +++ b/lib/api/bucketDeleteLifecycle.js @@ -28,8 +28,7 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) { error: err, method: 'bucketDeleteLifecycle', }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketLifecycle'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketLifecycle'); return callback(err, corsHeaders); } if (!bucket.getLifecycleConfiguration()) { @@ -46,16 +45,14 @@ function bucketDeleteLifecycle(authInfo, request, log, callback) { bucket.setLifecycleConfiguration(null); return metadata.updateBucket(bucketName, bucket, log, err => { if (err) { - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketLifecycle'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketLifecycle'); return callback(err, corsHeaders); } pushMetric('deleteBucketLifecycle', log, { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteBucketLifecycle'); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteBucketLifecycle'); return callback(null, corsHeaders); }); }); diff --git a/lib/api/bucketDeleteQuota.js b/lib/api/bucketDeleteQuota.js index 849072a45f..6584ce62df 100644 --- a/lib/api/bucketDeleteQuota.js +++ b/lib/api/bucketDeleteQuota.js @@ -25,34 +25,35 @@ function bucketDeleteQuota(authInfo, request, log, callback) { requestType: request.apiMethods || requestType, request, }; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket)), - (bucket, next) => { - bucket.setQuota(0); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketDeleteQuota' + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket), + ), + (bucket, next) => { + bucket.setQuota(0); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, + method: 'bucketDeleteQuota', + }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'bucketDeleteQuota'); + return callback(err, err.code, corsHeaders); + } + monitoring.promMetrics('DELETE', bucketName, '204', 'bucketDeleteQuota'); + pushMetric('bucketDeleteQuota', log, { + authInfo, + bucket: bucketName, }); - monitoring.promMetrics('DELETE', bucketName, err.code, - 'bucketDeleteQuota'); - return callback(err, err.code, corsHeaders); - } - monitoring.promMetrics( - 'DELETE', bucketName, '204', 'bucketDeleteQuota'); - pushMetric('bucketDeleteQuota', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, 204, corsHeaders); - }); + return callback(null, 204, corsHeaders); + }, + ); } module.exports = bucketDeleteQuota; diff --git a/lib/api/bucketDeleteReplication.js b/lib/api/bucketDeleteReplication.js index ff9912171e..220284ffcb 100644 --- a/lib/api/bucketDeleteReplication.js +++ b/lib/api/bucketDeleteReplication.js @@ -28,8 +28,7 @@ function bucketDeleteReplication(authInfo, request, log, callback) { error: err, method: 'bucketDeleteReplication', }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketReplication'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketReplication'); return callback(err, corsHeaders); } if (!bucket.getReplicationConfiguration()) { @@ -46,16 +45,14 @@ function bucketDeleteReplication(authInfo, request, log, callback) { bucket.setReplicationConfiguration(null); return metadata.updateBucket(bucketName, bucket, log, err => { if (err) { - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteBucketReplication'); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketReplication'); return callback(err, corsHeaders); } pushMetric('deleteBucketReplication', log, { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteBucketReplication'); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteBucketReplication'); return callback(null, corsHeaders); }); }); diff --git a/lib/api/bucketDeleteTagging.js b/lib/api/bucketDeleteTagging.js index 8997d760ef..0ac35e5082 100644 --- a/lib/api/bucketDeleteTagging.js +++ b/lib/api/bucketDeleteTagging.js @@ -25,38 +25,38 @@ function bucketDeleteTagging(authInfo, request, log, callback) { }; let bucket = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - if (err) { - return next(err); - } - bucket = b; - bucket.setTags([]); - return next(); - }), - next => metadata.updateBucket(bucket.getName(), bucket, log, next), - ], err => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.error('error processing request', { - error: err, - method: 'deleteBucketTagging', - bucketName + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, b) => { + if (err) { + return next(err); + } + bucket = b; + bucket.setTags([]); + return next(); + }), + next => metadata.updateBucket(bucket.getName(), bucket, log, next), + ], + err => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.error('error processing request', { + error: err, + method: 'deleteBucketTagging', + bucketName, + }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteBucketTagging'); + return callback(err, corsHeaders); + } + pushMetric('deleteBucketTagging', log, { + authInfo, + bucket: bucketName, }); - monitoring.promMetrics('DELETE', bucketName, err.code, - 'deleteBucketTagging'); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteBucketTagging'); return callback(err, corsHeaders); - } - pushMetric('deleteBucketTagging', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteBucketTagging'); - return callback(err, corsHeaders); - }); + }, + ); } module.exports = bucketDeleteTagging; diff --git a/lib/api/bucketGet.js b/lib/api/bucketGet.js index 42689a6d17..9f1e994e5e 100644 --- a/lib/api/bucketGet.js +++ b/lib/api/bucketGet.js @@ -106,7 +106,9 @@ function processVersions(bucketName, listParams, list) { xml.push( '', '', - '', bucketName, '' + '', + bucketName, + '', ); const isTruncated = list.IsTruncated ? 'true' : 'false'; const xmlParams = [ @@ -124,9 +126,8 @@ function processVersions(bucketName, listParams, list) { const escapeXmlFn = listParams.encoding === 'url' ? querystring.escape : escapeForXml; xmlParams.forEach(p => { if (p.value) { - const val = p.tag !== 'NextVersionIdMarker' || p.value === 'null' ? - p.value : - versionIdUtils.encode(p.value); + const val = + p.tag !== 'NextVersionIdMarker' || p.value === 'null' ? p.value : versionIdUtils.encode(p.value); xml.push(`<${p.tag}>${escapeXmlFn(val)}`); } }); @@ -141,9 +142,7 @@ function processVersions(bucketName, listParams, list) { v.IsDeleteMarker ? '' : '', `${objectKey}`, '', - (v.IsNull || v.VersionId === undefined) ? - 'null' - : versionIdUtils.encode(v.VersionId), + v.IsNull || v.VersionId === undefined ? 'null' : versionIdUtils.encode(v.VersionId), '', `${isLatest}`, `${v.LastModified}`, @@ -159,10 +158,7 @@ function processVersions(bucketName, listParams, list) { buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml); } - xml.push( - `${v.StorageClass}`, - v.IsDeleteMarker ? '' : '' - ); + xml.push(`${v.StorageClass}`, v.IsDeleteMarker ? '' : ''); }); list.CommonPrefixes.forEach(item => { @@ -179,7 +175,9 @@ function processMasterVersions(bucketName, listParams, list) { xml.push( '', '', - '', bucketName, '' + '', + bucketName, + '', ); const isTruncated = list.IsTruncated ? 'true' : 'false'; const xmlParams = [ @@ -207,11 +205,13 @@ function processMasterVersions(bucketName, listParams, list) { xml.push(`<${p.tag}>${p.value}`); } else if (p.value || p.tag === 'KeyCount' || p.tag === 'MaxKeys') { xml.push(`<${p.tag}>${escapeXmlFn(p.value)}`); - } else if (p.tag !== 'NextMarker' && - p.tag !== 'EncodingType' && - p.tag !== 'Delimiter' && - p.tag !== 'StartAfter' && - p.tag !== 'NextContinuationToken') { + } else if ( + p.tag !== 'NextMarker' && + p.tag !== 'EncodingType' && + p.tag !== 'Delimiter' && + p.tag !== 'StartAfter' && + p.tag !== 'NextContinuationToken' + ) { xml.push(`<${p.tag}/>`); } }); @@ -228,7 +228,7 @@ function processMasterVersions(bucketName, listParams, list) { `${objectKey}`, `${v.LastModified}`, `"${v.ETag}"`, - `${v.Size}` + `${v.Size}`, ); if (!listParams.v2 || listParams.fetchOwner) { @@ -236,16 +236,13 @@ function processMasterVersions(bucketName, listParams, list) { '', `${v.Owner.ID}`, `${v.Owner.DisplayName}`, - '' + '', ); } buildAttributesXml(v, v.userMetadata, listParams.optionalAttributes, xml); - return xml.push( - `${v.StorageClass}`, - '' - ); + return xml.push(`${v.StorageClass}`, ''); }); list.CommonPrefixes.forEach(item => { xml.push(`${escapeXmlFn(item)}`); @@ -375,9 +372,9 @@ async function bucketGet(authInfo, request, log, callback) { listParams.listingType = 'DelimiterVersions'; delete listParams.marker; listParams.keyMarker = params['key-marker']; - listParams.versionIdMarker = params['version-id-marker'] ? - versionIdUtils.decode(params['version-id-marker']) : - undefined; + listParams.versionIdMarker = params['version-id-marker'] + ? versionIdUtils.decode(params['version-id-marker']) + : undefined; } if (!requestMaxKeys) { const emptyList = { diff --git a/lib/api/bucketGetACL.js b/lib/api/bucketGetACL.js index 0726e06549..7eb47ced9a 100644 --- a/lib/api/bucketGetACL.js +++ b/lib/api/bucketGetACL.js @@ -26,7 +26,6 @@ const monitoring = require('../utilities/monitoringHandler'); */ - /** * bucketGetACL - Return ACL's for bucket * @param {AuthInfo} authInfo - Instance of AuthInfo class with requester's info @@ -56,13 +55,10 @@ function bucketGetACL(authInfo, request, log, callback) { }; standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.debug('error processing request', - { method: 'bucketGetACL', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketAcl'); + log.debug('error processing request', { method: 'bucketGetACL', error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketAcl'); return callback(err, null, corsHeaders); } const bucketACL = bucket.getAcl(); @@ -75,8 +71,7 @@ function bucketGetACL(authInfo, request, log, callback) { }; if (bucketACL.Canned !== '') { - const cannedGrants = aclUtils.handleCannedGrant( - bucketACL.Canned, ownerGrant); + const cannedGrants = aclUtils.handleCannedGrant(bucketACL.Canned, ownerGrant); grantInfo.grants = grantInfo.grants.concat(cannedGrants); const xml = aclUtils.convertToXml(grantInfo); pushMetric('getBucketAcl', log, { @@ -86,19 +81,19 @@ function bucketGetACL(authInfo, request, log, callback) { return callback(null, xml, corsHeaders); } /** - * Build array of all canonicalIDs used in ACLs so duplicates - * will be retained (e.g. if an account has both read and write - * privileges, want to display both and not lose the duplicate - * when receive one dictionary entry back from Vault) - */ + * Build array of all canonicalIDs used in ACLs so duplicates + * will be retained (e.g. if an account has both read and write + * privileges, want to display both and not lose the duplicate + * when receive one dictionary entry back from Vault) + */ const canonicalIDs = aclUtils.getCanonicalIDs(bucketACL); // Build array with grants by URI const uriGrantInfo = aclUtils.getUriGrantInfo(bucketACL); if (canonicalIDs.length === 0) { /** - * If no acl's set by account canonicalID, just add URI - * grants (if any) and return - */ + * If no acl's set by account canonicalID, just add URI + * grants (if any) and return + */ grantInfo.grants = grantInfo.grants.concat(uriGrantInfo); const xml = aclUtils.convertToXml(grantInfo); pushMetric('getBucketAcl', log, { @@ -108,22 +103,18 @@ function bucketGetACL(authInfo, request, log, callback) { return callback(null, xml, corsHeaders); } /** - * If acl's set by account canonicalID, get emails from Vault to serve - * as display names - */ + * If acl's set by account canonicalID, get emails from Vault to serve + * as display names + */ return vault.getEmailAddresses(canonicalIDs, log, (err, emails) => { if (err) { - log.debug('error processing request', - { method: 'vault.getEmailAddresses', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketAcl'); + log.debug('error processing request', { method: 'vault.getEmailAddresses', error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketAcl'); return callback(err, null, corsHeaders); } - const individualGrants = - aclUtils.getIndividualGrants(bucketACL, canonicalIDs, emails); + const individualGrants = aclUtils.getIndividualGrants(bucketACL, canonicalIDs, emails); // Add to grantInfo any individual grants and grants by uri - grantInfo.grants = grantInfo.grants - .concat(individualGrants).concat(uriGrantInfo); + grantInfo.grants = grantInfo.grants.concat(individualGrants).concat(uriGrantInfo); // parse info about accounts and owner info to convert to xml const xml = aclUtils.convertToXml(grantInfo); pushMetric('getBucketAcl', log, { diff --git a/lib/api/bucketGetEncryption.js b/lib/api/bucketGetEncryption.js index db5d31432b..a71c3c90f8 100644 --- a/lib/api/bucketGetEncryption.js +++ b/lib/api/bucketGetEncryption.js @@ -28,61 +28,65 @@ function bucketGetEncryption(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), - (bucket, next) => { - // If sseInfo is present but the `mandatory` flag is not set - // then this info was not created using bucketPutEncryption - // or by using the x-amz-scal-server-side-encryption header at - // bucket creation and should not be returned - const sseInfo = bucket.getServerSideEncryption(); - if (sseInfo === null || !sseInfo.mandatory) { - log.trace('no server side encryption config found', { - bucket: bucketName, - method: 'bucketGetEncryption', - }); - return next(errors.ServerSideEncryptionConfigurationNotFoundError); + return async.waterfall( + [ + next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), + (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), + (bucket, next) => { + // If sseInfo is present but the `mandatory` flag is not set + // then this info was not created using bucketPutEncryption + // or by using the x-amz-scal-server-side-encryption header at + // bucket creation and should not be returned + const sseInfo = bucket.getServerSideEncryption(); + if (sseInfo === null || !sseInfo.mandatory) { + log.trace('no server side encryption config found', { + bucket: bucketName, + method: 'bucketGetEncryption', + }); + return next(errors.ServerSideEncryptionConfigurationNotFoundError); + } + return next(null, bucket, sseInfo); + }, + ], + (error, bucket, sseInfo) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (error) { + return callback(error, corsHeaders); } - return next(null, bucket, sseInfo); - }, - ], - (error, bucket, sseInfo) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (error) { - return callback(error, corsHeaders); - } - const xml = [ - '', - '', - '', - '', - `${escapeForXml(sseInfo.algorithm)}`, - ]; + const xml = [ + '', + '', + '', + '', + `${escapeForXml(sseInfo.algorithm)}`, + ]; - if (sseInfo.configuredMasterKeyId) { - xml.push(`${escapeForXml( - config.kmsHideScalityArn - ? getKeyIdFromArn(sseInfo.configuredMasterKeyId) - : sseInfo.configuredMasterKeyId - )}`); - } + if (sseInfo.configuredMasterKeyId) { + xml.push( + `${escapeForXml( + config.kmsHideScalityArn + ? getKeyIdFromArn(sseInfo.configuredMasterKeyId) + : sseInfo.configuredMasterKeyId, + )}`, + ); + } - xml.push( - '', - 'false', - '', - '' - ); + xml.push( + '', + 'false', + '', + '', + ); - pushMetric('getBucketEncryption', log, { - authInfo, - bucket: bucketName, - }); + pushMetric('getBucketEncryption', log, { + authInfo, + bucket: bucketName, + }); - return callback(null, xml.join(''), corsHeaders); - }); + return callback(null, xml.join(''), corsHeaders); + }, + ); } module.exports = bucketGetEncryption; diff --git a/lib/api/bucketGetLifecycle.js b/lib/api/bucketGetLifecycle.js index bc2851172c..8a8e5bca72 100644 --- a/lib/api/bucketGetLifecycle.js +++ b/lib/api/bucketGetLifecycle.js @@ -1,6 +1,5 @@ const { errors } = require('arsenal'); -const LifecycleConfiguration = - require('arsenal').models.LifecycleConfiguration; +const LifecycleConfiguration = require('arsenal').models.LifecycleConfiguration; const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -31,8 +30,7 @@ function bucketGetLifecycle(authInfo, request, log, callback) { error: err, method: 'bucketGetLifecycle', }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketLifecycle'); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketLifecycle'); return callback(err, null, corsHeaders); } const lifecycleConfig = bucket.getLifecycleConfiguration(); @@ -41,10 +39,8 @@ function bucketGetLifecycle(authInfo, request, log, callback) { error: errors.NoSuchLifecycleConfiguration, method: 'bucketGetLifecycle', }); - monitoring.promMetrics( - 'GET', bucketName, 404, 'getBucketLifecycle'); - return callback(errors.NoSuchLifecycleConfiguration, null, - corsHeaders); + monitoring.promMetrics('GET', bucketName, 404, 'getBucketLifecycle'); + return callback(errors.NoSuchLifecycleConfiguration, null, corsHeaders); } const xml = LifecycleConfiguration.getConfigXml(lifecycleConfig); pushMetric('getBucketLifecycle', log, { diff --git a/lib/api/bucketGetLocation.js b/lib/api/bucketGetLocation.js index 31d8b6dfec..8e4c7db549 100644 --- a/lib/api/bucketGetLocation.js +++ b/lib/api/bucketGetLocation.js @@ -40,7 +40,8 @@ function bucketGetLocation(authInfo, request, log, callback) { // if no locationConstraint provided. locationConstraint = ''; } - const xml = ` + const xml = + ` ` + `${escapeForXml(locationConstraint)}`; pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); diff --git a/lib/api/bucketGetLogging.js b/lib/api/bucketGetLogging.js index 501caf8942..585d4f460f 100644 --- a/lib/api/bucketGetLogging.js +++ b/lib/api/bucketGetLogging.js @@ -5,7 +5,8 @@ const { waterfall } = require('async'); const { config, serverAccessLogsModes } = require('../Config'); const { errorInstances } = require('arsenal'); -const BucketLoggingStatusNotFoundBody = '\n' + +const BucketLoggingStatusNotFoundBody = + '\n' + ''; function bucketGetLogging(authInfo, request, log, callback) { @@ -23,39 +24,44 @@ function bucketGetLogging(authInfo, request, log, callback) { request, }; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - if (err) { - return next(err); - } + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err); + } - return next(null, bucket); - }), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => { - if (err) { - return next(err); - } + return next(null, bucket); + }), + (bucket, next) => + checkExpectedBucketOwner(request.headers, bucket, log, err => { + if (err) { + return next(err); + } + + return next(null, bucket); + }), + (bucket, next) => { + const bucketLoggingStatus = bucket.getBucketLoggingStatus(); + if (!bucketLoggingStatus) { + return next(null, BucketLoggingStatusNotFoundBody); + } - return next(null, bucket); - }), - (bucket, next) => { - const bucketLoggingStatus = bucket.getBucketLoggingStatus(); - if (!bucketLoggingStatus) { - return next(null, BucketLoggingStatusNotFoundBody); + return next(null, bucketLoggingStatus.toXML()); + }, + ], + (err, body) => { + if (err) { + log.trace('error processing request', { error: err, method: 'bucketGetLogging' }); + monitoring.promMetrics('GET', bucketName, err.code, 'bucketGetLogging'); + return callback(err); } - return next(null, bucketLoggingStatus.toXML()); - } - ], (err, body) => { - if (err) { - log.trace('error processing request', { error: err, method: 'bucketGetLogging' }); - monitoring.promMetrics('GET', bucketName, err.code, 'bucketGetLogging'); - return callback(err); - } - - monitoring.promMetrics('GET', bucketName, '200', 'bucketGetLogging'); - return callback(null, body); - }); + monitoring.promMetrics('GET', bucketName, '200', 'bucketGetLogging'); + return callback(null, body); + }, + ); } module.exports = bucketGetLogging; diff --git a/lib/api/bucketGetObjectLock.js b/lib/api/bucketGetObjectLock.js index a96e7cb4e4..e2aab76c56 100644 --- a/lib/api/bucketGetObjectLock.js +++ b/lib/api/bucketGetObjectLock.js @@ -2,8 +2,7 @@ const { errors } = require('arsenal'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const ObjectLockConfiguration = - require('arsenal').models.ObjectLockConfiguration; +const ObjectLockConfiguration = require('arsenal').models.ObjectLockConfiguration; // Format of the xml response: /** @@ -53,8 +52,7 @@ function bucketGetObjectLock(authInfo, request, log, callback) { error: errors.ObjectLockConfigurationNotFoundError, method: 'bucketGetObjectLock', }); - return callback(errors.ObjectLockConfigurationNotFoundError, null, - corsHeaders); + return callback(errors.ObjectLockConfigurationNotFoundError, null, corsHeaders); } const xml = ObjectLockConfiguration.getConfigXML(objLockConfig); pushMetric('getBucketObjectLock', log, { diff --git a/lib/api/bucketGetPolicy.js b/lib/api/bucketGetPolicy.js index d6f3b6a576..1f521dc1d9 100644 --- a/lib/api/bucketGetPolicy.js +++ b/lib/api/bucketGetPolicy.js @@ -36,8 +36,7 @@ function bucketGetPolicy(authInfo, request, log, callback) { error: errors.NoSuchBucketPolicy, method: 'bucketGetPolicy', }); - return callback(errors.NoSuchBucketPolicy, null, - corsHeaders); + return callback(errors.NoSuchBucketPolicy, null, corsHeaders); } // TODO: implement Utapi metric support // bucketPolicy needs to be JSON stringified on return for proper diff --git a/lib/api/bucketGetQuota.js b/lib/api/bucketGetQuota.js index 9556e31d4d..963eaecacb 100644 --- a/lib/api/bucketGetQuota.js +++ b/lib/api/bucketGetQuota.js @@ -31,21 +31,15 @@ function bucketGetQuota(authInfo, request, log, callback) { }); return callback(err, null, corsHeaders); } - xml.push( - '', - '', - '', bucket.getName(), '', - ); + xml.push('', '', '', bucket.getName(), ''); const bucketQuota = bucket.getQuota(); if (!bucketQuota) { log.debug('bucket has no quota', { method: 'bucketGetQuota', }); - return callback(errors.NoSuchQuota, null, - corsHeaders); + return callback(errors.NoSuchQuota, null, corsHeaders); } - xml.push('', bucketQuota, '', - ''); + xml.push('', bucketQuota, '', ''); pushMetric('getBucketQuota', log, { authInfo, diff --git a/lib/api/bucketGetRateLimit.js b/lib/api/bucketGetRateLimit.js index 8372e9afd4..48538a604b 100644 --- a/lib/api/bucketGetRateLimit.js +++ b/lib/api/bucketGetRateLimit.js @@ -43,8 +43,7 @@ function bucketGetRateLimit(authInfo, request, log, callback) { error: errors.NoSuchRateLimitConfig, method: 'bucketGetRateLimit', }); - return callback(errors.NoSuchRateLimitConfig, null, - corsHeaders); + return callback(errors.NoSuchRateLimitConfig, null, corsHeaders); } return callback(null, JSON.stringify(rateLimitConfig.getData()), corsHeaders); diff --git a/lib/api/bucketGetReplication.js b/lib/api/bucketGetReplication.js index 8891f96bc1..e1a75a1e20 100644 --- a/lib/api/bucketGetReplication.js +++ b/lib/api/bucketGetReplication.js @@ -2,8 +2,7 @@ const { errors } = require('arsenal'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); -const { getReplicationConfigurationXML } = - require('./apiUtils/bucket/getReplicationConfiguration'); +const { getReplicationConfigurationXML } = require('./apiUtils/bucket/getReplicationConfiguration'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const monitoring = require('../utilities/monitoringHandler'); @@ -31,8 +30,7 @@ function bucketGetReplication(authInfo, request, log, callback) { error: err, method: 'bucketGetReplication', }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketReplication'); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketReplication'); return callback(err, null, corsHeaders); } const replicationConfig = bucket.getReplicationConfiguration(); @@ -41,18 +39,15 @@ function bucketGetReplication(authInfo, request, log, callback) { error: errors.ReplicationConfigurationNotFoundError, method: 'bucketGetReplication', }); - monitoring.promMetrics( - 'GET', bucketName, 404, 'getBucketReplication'); - return callback(errors.ReplicationConfigurationNotFoundError, null, - corsHeaders); + monitoring.promMetrics('GET', bucketName, 404, 'getBucketReplication'); + return callback(errors.ReplicationConfigurationNotFoundError, null, corsHeaders); } const xml = getReplicationConfigurationXML(replicationConfig); pushMetric('getBucketReplication', log, { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getBucketReplication'); + monitoring.promMetrics('GET', bucketName, '200', 'getBucketReplication'); return callback(null, xml, corsHeaders); }); } diff --git a/lib/api/bucketGetTagging.js b/lib/api/bucketGetTagging.js index c31a14a08a..83c4de8e45 100644 --- a/lib/api/bucketGetTagging.js +++ b/lib/api/bucketGetTagging.js @@ -74,44 +74,44 @@ function bucketGetTagging(authInfo, request, log, callback) { let xml = null; let tags = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - bucket = b; - return next(err); - }), - next => checkExpectedBucketOwner(headers, bucket, log, next), - next => { - tags = bucket.getTags(); - if (!tags || !tags.length) { - log.debug('bucket TagSet does not exist', { + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, b) => { + bucket = b; + return next(err); + }), + next => checkExpectedBucketOwner(headers, bucket, log, next), + next => { + tags = bucket.getTags(); + if (!tags || !tags.length) { + log.debug('bucket TagSet does not exist', { + method: 'bucketGetTagging', + }); + return next(errors.NoSuchTagSet); + } + xml = tagsToXml(tags); + return next(); + }, + ], + err => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, method: 'bucketGetTagging', }); - return next(errors.NoSuchTagSet); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketTagging'); + } else { + pushMetric('getBucketTagging', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('GET', bucketName, '200', 'getBucketTagging'); } - xml = tagsToXml(tags); - return next(); - } - ], err => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketGetTagging' - }); - monitoring.promMetrics('GET', bucketName, err.code, - 'getBucketTagging'); - } else { - pushMetric('getBucketTagging', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getBucketTagging'); - } - return callback(err, xml, corsHeaders); - }); + return callback(err, xml, corsHeaders); + }, + ); } module.exports = bucketGetTagging; diff --git a/lib/api/bucketGetVersioning.js b/lib/api/bucketGetVersioning.js index f38cc31053..101d0eb8b7 100644 --- a/lib/api/bucketGetVersioning.js +++ b/lib/api/bucketGetVersioning.js @@ -19,9 +19,9 @@ const monitoring = require('../utilities/monitoringHandler'); function convertToXml(versioningConfiguration) { const xml = []; - xml.push('', - '' + xml.push( + '', + '', ); if (versioningConfiguration && versioningConfiguration.Status) { @@ -59,13 +59,10 @@ function bucketGetVersioning(authInfo, request, log, callback) { }; standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.debug('error processing request', - { method: 'bucketGetVersioning', error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getBucketVersioning'); + log.debug('error processing request', { method: 'bucketGetVersioning', error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getBucketVersioning'); return callback(err, null, corsHeaders); } const versioningConfiguration = bucket.getVersioningConfiguration(); @@ -74,8 +71,7 @@ function bucketGetVersioning(authInfo, request, log, callback) { authInfo, bucket: bucketName, }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getBucketVersioning'); + monitoring.promMetrics('GET', bucketName, '200', 'getBucketVersioning'); return callback(null, xml, corsHeaders); }); } diff --git a/lib/api/bucketHead.js b/lib/api/bucketHead.js index bafc52eb2d..506b3f9986 100644 --- a/lib/api/bucketHead.js +++ b/lib/api/bucketHead.js @@ -23,11 +23,9 @@ function bucketHead(authInfo, request, log, callback) { request, }; standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headBucket'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headBucket'); return callback(err, corsHeaders); } pushMetric('headBucket', log, { diff --git a/lib/api/bucketPutACL.js b/lib/api/bucketPutACL.js index a6e86a4dba..2a2134c6ef 100644 --- a/lib/api/bucketPutACL.js +++ b/lib/api/bucketPutACL.js @@ -54,18 +54,14 @@ function bucketPutACL(authInfo, request, log, callback) { 'authenticated-read', 'log-delivery-write', ]; - const possibleGroups = [constants.allAuthedUsersId, - constants.publicId, - constants.logId, - ]; + const possibleGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId]; const metadataValParams = { authInfo, bucketName, requestType: request.apiMethods || 'bucketPutACL', request, }; - const possibleGrants = ['FULL_CONTROL', 'WRITE', - 'WRITE_ACP', 'READ', 'READ_ACP']; + const possibleGrants = ['FULL_CONTROL', 'WRITE', 'WRITE_ACP', 'READ', 'READ_ACP']; const addACLParams = { Canned: '', FULL_CONTROL: [], @@ -75,235 +71,226 @@ function bucketPutACL(authInfo, request, log, callback) { READ_ACP: [], }; - const grantReadHeader = - aclUtils.parseGrant(request.headers[ - 'x-amz-grant-read'], 'READ'); - const grantWriteHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-write'], 'WRITE'); - const grantReadACPHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], - 'READ_ACP'); - const grantWriteACPHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-write-acp'], - 'WRITE_ACP'); - const grantFullControlHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-full-control'], - 'FULL_CONTROL'); + const grantReadHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read'], 'READ'); + const grantWriteHeader = aclUtils.parseGrant(request.headers['x-amz-grant-write'], 'WRITE'); + const grantReadACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(request.headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - return async.waterfall([ - function waterfall1(next) { - standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - log.trace('request authorization failed', { - error: err, - method: 'metadataValidateBucket', - }); - return next(err, bucket); - } - // if the API call is allowed, ensure that the parameters are valid - if (newCannedACL && possibleCannedACL.indexOf(newCannedACL) === -1) { - log.trace('invalid canned acl argument', { - acl: newCannedACL, - method: 'bucketPutACL', - }); - return next(errors.InvalidArgument); - } - if (!aclUtils.checkGrantHeaderValidity(request.headers)) { - log.trace('invalid acl header'); - return next(errors.InvalidArgument); - } - return next(null, bucket); - }); - }, - function waterfall2(bucket, next) { - // If not setting acl through headers, parse body - if (newCannedACL === undefined - && grantReadHeader === undefined - && grantWriteHeader === undefined - && grantReadACPHeader === undefined - && grantWriteACPHeader === undefined - && grantFullControlHeader === undefined) { - if (request.post) { - log.trace('parsing acls from request body'); - return aclUtils.parseAclXml(request.post, log, - (err, jsonGrants) => next(err, bucket, jsonGrants)); - } - // If no ACLs sent with request at all - return next(errors.MalformedXML, bucket); - } - /** - * If acl set in headers (including canned acl) pass bucket and - * undefined to the next function - */ - log.trace('using acls from request headers'); - return next(null, bucket, undefined); - }, - function waterfall3(bucket, jsonGrants, next) { - // If canned ACL just move on and set them - if (newCannedACL) { - log.trace('canned acl', { cannedAcl: newCannedACL }); - addACLParams.Canned = newCannedACL; - return next(null, bucket, addACLParams); - } - let usersIdentifiedByEmail = []; - let usersIdentifiedByGroup = []; - let usersIdentifiedByID = []; - let hasError = false; - /** - * If grants set by xml, loop through the grants - * and separate grant types so parsed in same manner - * as header grants - */ - if (jsonGrants) { - log.trace('parsing acl grants'); - jsonGrants.forEach(grant => { - const grantee = grant.Grantee[0]; - const granteeType = grantee.$['xsi:type']; - const permission = grant.Permission[0]; - let skip = false; - if (possibleGrants.indexOf(permission) < 0) { - skip = true; - } - if (!skip && granteeType === 'AmazonCustomerByEmail') { - usersIdentifiedByEmail.push({ - identifier: grantee.EmailAddress[0], - grantType: permission, - userIDType: 'emailaddress', + return async.waterfall( + [ + function waterfall1(next) { + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + log.trace('request authorization failed', { + error: err, + method: 'metadataValidateBucket', }); + return next(err, bucket); } - if (!skip && granteeType === 'CanonicalUser') { - usersIdentifiedByID.push({ - identifier: grantee.ID[0], - grantType: permission, - userIDType: 'id', + // if the API call is allowed, ensure that the parameters are valid + if (newCannedACL && possibleCannedACL.indexOf(newCannedACL) === -1) { + log.trace('invalid canned acl argument', { + acl: newCannedACL, + method: 'bucketPutACL', }); + return next(errors.InvalidArgument); } - if (!skip && granteeType === 'Group') { - if (possibleGroups.indexOf(grantee.URI[0]) < 0) { - log.trace('invalid user group', - { userGroup: grantee.URI[0] }); - hasError = true; - return next(errors.InvalidArgument, bucket); - } - return usersIdentifiedByGroup.push({ - identifier: grantee.URI[0], - grantType: permission, - userIDType: 'uri', - }); + if (!aclUtils.checkGrantHeaderValidity(request.headers)) { + log.trace('invalid acl header'); + return next(errors.InvalidArgument); } - return undefined; + return next(null, bucket); }); - if (hasError) { - return undefined; + }, + function waterfall2(bucket, next) { + // If not setting acl through headers, parse body + if ( + newCannedACL === undefined && + grantReadHeader === undefined && + grantWriteHeader === undefined && + grantReadACPHeader === undefined && + grantWriteACPHeader === undefined && + grantFullControlHeader === undefined + ) { + if (request.post) { + log.trace('parsing acls from request body'); + return aclUtils.parseAclXml(request.post, log, (err, jsonGrants) => + next(err, bucket, jsonGrants), + ); + } + // If no ACLs sent with request at all + return next(errors.MalformedXML, bucket); } - } else { - // If no canned ACL and no parsed xml, loop - // through the access headers - const allGrantHeaders = - [].concat(grantReadHeader, grantWriteHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader); + /** + * If acl set in headers (including canned acl) pass bucket and + * undefined to the next function + */ + log.trace('using acls from request headers'); + return next(null, bucket, undefined); + }, + function waterfall3(bucket, jsonGrants, next) { + // If canned ACL just move on and set them + if (newCannedACL) { + log.trace('canned acl', { cannedAcl: newCannedACL }); + addACLParams.Canned = newCannedACL; + return next(null, bucket, addACLParams); + } + let usersIdentifiedByEmail = []; + let usersIdentifiedByGroup = []; + let usersIdentifiedByID = []; + let hasError = false; + /** + * If grants set by xml, loop through the grants + * and separate grant types so parsed in same manner + * as header grants + */ + if (jsonGrants) { + log.trace('parsing acl grants'); + jsonGrants.forEach(grant => { + const grantee = grant.Grantee[0]; + const granteeType = grantee.$['xsi:type']; + const permission = grant.Permission[0]; + let skip = false; + if (possibleGrants.indexOf(permission) < 0) { + skip = true; + } + if (!skip && granteeType === 'AmazonCustomerByEmail') { + usersIdentifiedByEmail.push({ + identifier: grantee.EmailAddress[0], + grantType: permission, + userIDType: 'emailaddress', + }); + } + if (!skip && granteeType === 'CanonicalUser') { + usersIdentifiedByID.push({ + identifier: grantee.ID[0], + grantType: permission, + userIDType: 'id', + }); + } + if (!skip && granteeType === 'Group') { + if (possibleGroups.indexOf(grantee.URI[0]) < 0) { + log.trace('invalid user group', { userGroup: grantee.URI[0] }); + hasError = true; + return next(errors.InvalidArgument, bucket); + } + return usersIdentifiedByGroup.push({ + identifier: grantee.URI[0], + grantType: permission, + userIDType: 'uri', + }); + } + return undefined; + }); + if (hasError) { + return undefined; + } + } else { + // If no canned ACL and no parsed xml, loop + // through the access headers + const allGrantHeaders = [].concat( + grantReadHeader, + grantWriteHeader, + grantReadACPHeader, + grantWriteACPHeader, + grantFullControlHeader, + ); - usersIdentifiedByEmail = allGrantHeaders.filter(item => - item && item.userIDType.toLowerCase() === 'emailaddress'); + usersIdentifiedByEmail = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'emailaddress', + ); - usersIdentifiedByGroup = allGrantHeaders - .filter(itm => itm && itm.userIDType - .toLowerCase() === 'uri'); - for (let i = 0; i < usersIdentifiedByGroup.length; i++) { - const userGroup = usersIdentifiedByGroup[i].identifier; - if (possibleGroups.indexOf(userGroup) < 0) { - log.trace('invalid user group', { userGroup, - method: 'bucketPutACL' }); - return next(errors.InvalidArgument, bucket); + usersIdentifiedByGroup = allGrantHeaders.filter( + itm => itm && itm.userIDType.toLowerCase() === 'uri', + ); + for (let i = 0; i < usersIdentifiedByGroup.length; i++) { + const userGroup = usersIdentifiedByGroup[i].identifier; + if (possibleGroups.indexOf(userGroup) < 0) { + log.trace('invalid user group', { userGroup, method: 'bucketPutACL' }); + return next(errors.InvalidArgument, bucket); + } } + /** TODO: Consider whether want to verify with Vault + * whether canonicalID is associated with existing + * account before adding to ACL */ + usersIdentifiedByID = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'id', + ); } - /** TODO: Consider whether want to verify with Vault - * whether canonicalID is associated with existing - * account before adding to ACL */ - usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType - .toLowerCase() === 'id'); - } - // For now, at least make sure ID is 64-char alphanumeric - // string before adding to ACL (check can be removed if - // verifying with Vault for associated accounts first) - for (let i = 0; i < usersIdentifiedByID.length; i++) { - const id = usersIdentifiedByID[i].identifier; - if (!aclUtils.isValidCanonicalId(id)) { - log.trace('invalid user id argument', { - id, - method: 'bucketPutACL', - }); - monitoring.promMetrics('PUT', bucketName, 400, - 'bucketPutACL'); - return callback(errors.InvalidArgument, bucket); + // For now, at least make sure ID is 64-char alphanumeric + // string before adding to ACL (check can be removed if + // verifying with Vault for associated accounts first) + for (let i = 0; i < usersIdentifiedByID.length; i++) { + const id = usersIdentifiedByID[i].identifier; + if (!aclUtils.isValidCanonicalId(id)) { + log.trace('invalid user id argument', { + id, + method: 'bucketPutACL', + }); + monitoring.promMetrics('PUT', bucketName, 400, 'bucketPutACL'); + return callback(errors.InvalidArgument, bucket); + } } - } - const justEmails = usersIdentifiedByEmail - .map(item => item.identifier); - // If have to lookup canonicalID's do that asynchronously - if (justEmails.length > 0) { - return vault.getCanonicalIds(justEmails, log, - (err, results) => { + const justEmails = usersIdentifiedByEmail.map(item => item.identifier); + // If have to lookup canonicalID's do that asynchronously + if (justEmails.length > 0) { + return vault.getCanonicalIds(justEmails, log, (err, results) => { if (err) { log.trace('error looking up canonical ids', { - error: err, method: 'vault.getCanonicalIDs' }); + error: err, + method: 'vault.getCanonicalIDs', + }); return next(err, bucket); } - const reconstructedUsersIdentifiedByEmail = aclUtils - .reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = aclUtils - .sortHeaderGrants(allUsers, addACLParams); + usersIdentifiedByGroup, + ); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); return next(null, bucket, revisedAddACLParams); }); + } + const allUsers = [].concat(usersIdentifiedByID, usersIdentifiedByGroup); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); + return next(null, bucket, revisedAddACLParams); + }, + function waterfall4(bucket, addACLParams, next) { + if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { + log.trace('deleted flag on bucket'); + return next(errors.NoSuchBucket); + } + if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { + log.trace('transient or deleted flag so cleaning up bucket'); + bucket.setFullAcl(addACLParams); + return cleanUpBucket(bucket, canonicalID, log, err => next(err, bucket)); + } + // If no bucket flags, just add acl's to bucket metadata + return acl.addACL(bucket, addACLParams, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutACL' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutACL'); + } else { + pushMetric('putBucketAcl', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutACL'); } - const allUsers = [].concat( - usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = - aclUtils.sortHeaderGrants(allUsers, addACLParams); - return next(null, bucket, revisedAddACLParams); - }, - function waterfall4(bucket, addACLParams, next) { - if (bucket.hasDeletedFlag() && canonicalID !== bucket.getOwner()) { - log.trace('deleted flag on bucket'); - return next(errors.NoSuchBucket); - } - if (bucket.hasTransientFlag() || bucket.hasDeletedFlag()) { - log.trace('transient or deleted flag so cleaning up bucket'); - bucket.setFullAcl(addACLParams); - return cleanUpBucket(bucket, canonicalID, log, err => - next(err, bucket)); - } - // If no bucket flags, just add acl's to bucket metadata - return acl.addACL(bucket, addACLParams, log, err => - next(err, bucket)); + return callback(err, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutACL' }); - monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutACL'); - } else { - pushMetric('putBucketAcl', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutACL'); - } - return callback(err, corsHeaders); - }); + ); } module.exports = bucketPutACL; diff --git a/lib/api/bucketPutCors.js b/lib/api/bucketPutCors.js index 2cc05d25ad..e17cc53332 100644 --- a/lib/api/bucketPutCors.js +++ b/lib/api/bucketPutCors.js @@ -30,8 +30,7 @@ function bucketPutCors(authInfo, request, log, callback) { }; if (!request.post) { - log.debug('CORS xml body is missing', - { error: errors.MissingRequestBodyError }); + log.debug('CORS xml body is missing', { error: errors.MissingRequestBodyError }); monitoring.promMetrics('PUT', bucketName, 400, METRICS_ACTION); return callback(errors.MissingRequestBodyError); } @@ -43,38 +42,41 @@ function bucketPutCors(authInfo, request, log, callback) { return callback(errorInstances.MalformedXML.customizeDescription(errMsg)); } - return async.waterfall([ - next => { - log.trace('parsing cors rules'); - return parseCorsXml(request.post, log, next); - }, - (rules, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - if (err?.is?.AccessDenied) { - return next(err, corsHeaders); + return async.waterfall( + [ + next => { + log.trace('parsing cors rules'); + return parseCorsXml(request.post, log, next); + }, + (rules, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + if (err?.is?.AccessDenied) { + return next(err, corsHeaders); + } + return next(err); } - return next(err); - } - return next(null, bucket, rules, corsHeaders); - }), - (bucket, rules, corsHeaders, next) => { - bucket.setCors(rules); - return metadata.updateBucket(bucketName, bucket, log, err => next(err, corsHeaders)); + return next(null, bucket, rules, corsHeaders); + }), + (bucket, rules, corsHeaders, next) => { + bucket.setCors(rules); + return metadata.updateBucket(bucketName, bucket, log, err => next(err, corsHeaders)); + }, + ], + (err, corsHeaders) => { + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutCors' }); + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + return callback(err, corsHeaders); + } + pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); + monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); + return callback(null, corsHeaders); }, - ], (err, corsHeaders) => { - if (err) { - log.trace('error processing request', { error: err, method: 'bucketPutCors' }); - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - return callback(err, corsHeaders); - } - pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); - monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutCors; diff --git a/lib/api/bucketPutEncryption.js b/lib/api/bucketPutEncryption.js index e214407ed7..786fa5ad72 100644 --- a/lib/api/bucketPutEncryption.js +++ b/lib/api/bucketPutEncryption.js @@ -27,73 +27,74 @@ function bucketPutEncryption(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), - (bucket, next) => { - log.trace('parsing encryption config', { method: 'bucketPutEncryption' }); - return parseEncryptionXml(request.post, log, (err, encryptionConfig) => { - if (err) { - return next(err); - } - return next(null, bucket, encryptionConfig); - }); - }, - (bucket, encryptionConfig, next) => { - const existingConfig = bucket.getServerSideEncryption(); - // Check if encryption is not configured or if a default master key has not been created yet. - if (existingConfig === null || !existingConfig.masterKeyId) { - return kms.bucketLevelEncryption(bucket, encryptionConfig, log, - (err, updatedConfig) => { + return async.waterfall( + [ + next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, next), + (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => next(err, bucket)), + (bucket, next) => { + log.trace('parsing encryption config', { method: 'bucketPutEncryption' }); + return parseEncryptionXml(request.post, log, (err, encryptionConfig) => { + if (err) { + return next(err); + } + return next(null, bucket, encryptionConfig); + }); + }, + (bucket, encryptionConfig, next) => { + const existingConfig = bucket.getServerSideEncryption(); + // Check if encryption is not configured or if a default master key has not been created yet. + if (existingConfig === null || !existingConfig.masterKeyId) { + return kms.bucketLevelEncryption(bucket, encryptionConfig, log, (err, updatedConfig) => { if (err) { return next(err); } return next(null, bucket, updatedConfig); }); - } + } - // If encryption is already configured and a default master key exists + // If encryption is already configured and a default master key exists - // If the request does not specify a custom key, reuse the existing default master key id - // This ensures that a new default master key is not generated every time - // `putBucketEncryption` is called, avoiding unnecessary key creation - const updatedConfig = { - mandatory: true, - algorithm: encryptionConfig.algorithm, - cryptoScheme: existingConfig.cryptoScheme, - masterKeyId: existingConfig.masterKeyId, - }; + // If the request does not specify a custom key, reuse the existing default master key id + // This ensures that a new default master key is not generated every time + // `putBucketEncryption` is called, avoiding unnecessary key creation + const updatedConfig = { + mandatory: true, + algorithm: encryptionConfig.algorithm, + cryptoScheme: existingConfig.cryptoScheme, + masterKeyId: existingConfig.masterKeyId, + }; - // If the request specifies a custom master key id, store it in the updated configuration - const { configuredMasterKeyId } = encryptionConfig; - if (configuredMasterKeyId) { - updatedConfig.configuredMasterKeyId = configuredMasterKeyId; - } + // If the request specifies a custom master key id, store it in the updated configuration + const { configuredMasterKeyId } = encryptionConfig; + if (configuredMasterKeyId) { + updatedConfig.configuredMasterKeyId = configuredMasterKeyId; + } - const { isAccountEncryptionEnabled } = existingConfig; - if (isAccountEncryptionEnabled) { - updatedConfig.isAccountEncryptionEnabled = isAccountEncryptionEnabled; - } + const { isAccountEncryptionEnabled } = existingConfig; + if (isAccountEncryptionEnabled) { + updatedConfig.isAccountEncryptionEnabled = isAccountEncryptionEnabled; + } - return next(null, bucket, updatedConfig); - }, - (bucket, updatedConfig, next) => { - bucket.setServerSideEncryption(updatedConfig); - metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + return next(null, bucket, updatedConfig); + }, + (bucket, updatedConfig, next) => { + bucket.setServerSideEncryption(updatedConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutEncryption' }); + return callback(err, corsHeaders); + } + pushMetric('putBucketEncryption', log, { + authInfo, + bucket: bucketName, + }); + return callback(null, corsHeaders); }, - ], - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, method: 'bucketPutEncryption' }); - return callback(err, corsHeaders); - } - pushMetric('putBucketEncryption', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutEncryption; diff --git a/lib/api/bucketPutLogging.js b/lib/api/bucketPutLogging.js index 3592f4432a..0c51dfeb52 100644 --- a/lib/api/bucketPutLogging.js +++ b/lib/api/bucketPutLogging.js @@ -7,7 +7,7 @@ const monitoring = require('../utilities/monitoringHandler'); const { errorInstances } = require('arsenal'); const { config, serverAccessLogsModes } = require('../Config'); -const ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH = +const ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH = 'The owner for the bucket to be logged and the target bucket must be the same.'; function bucketPutLogging(authInfo, request, log, callback) { @@ -32,60 +32,68 @@ function bucketPutLogging(authInfo, request, log, callback) { request, }; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - if (err) { - return next(err); - } - - return next(null, bucket); - }), - (bucket, next) => { - const loggingEnabled = parsed.res.getLoggingEnabled(); - if (!loggingEnabled) { - return next(null, bucket); - } + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err); + } - return metadata.getBucket(loggingEnabled.TargetBucket, log, (err, targetBucket) => { - if (err) { - return next(errorInstances.InvalidTargetBucketForLogging.customizeDescription(err.description)); + return next(null, bucket); + }), + (bucket, next) => { + const loggingEnabled = parsed.res.getLoggingEnabled(); + if (!loggingEnabled) { + return next(null, bucket); } - if (targetBucket.getOwner() !== bucket.getOwner()) { - return next(errorInstances.InvalidTargetBucketForLogging - .customizeDescription(ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH)); - } + return metadata.getBucket(loggingEnabled.TargetBucket, log, (err, targetBucket) => { + if (err) { + return next(errorInstances.InvalidTargetBucketForLogging.customizeDescription(err.description)); + } - return next(null, bucket); - }); - }, - (bucket, next) => checkExpectedBucketOwner(request.headers, bucket, log, err => { + if (targetBucket.getOwner() !== bucket.getOwner()) { + return next( + errorInstances.InvalidTargetBucketForLogging.customizeDescription( + ERROR_MSG_SOURCE_TARGET_BUCKET_OWNER_MISMATCH, + ), + ); + } + + return next(null, bucket); + }); + }, + (bucket, next) => + checkExpectedBucketOwner(request.headers, bucket, log, err => { + if (err) { + return next(err); + } + + return next(null, bucket); + }), + (bucket, next) => { + bucket.setBucketLoggingStatus(parsed.res); + return metadata.updateBucket(bucket.getName(), bucket, log, err => { + if (err) { + return next(err); + } + + return next(); + }); + }, + ], + err => { if (err) { - return next(err); + log.trace('error processing request', { error: err, method: 'bucketPutLogging' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutLogging'); + return callback(err); } - return next(null, bucket); - }), - (bucket, next) => { - bucket.setBucketLoggingStatus(parsed.res); - return metadata.updateBucket(bucket.getName(), bucket, log, err => { - if (err) { - return next(err); - } - - return next(); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutLogging'); + return callback(); }, - ], err => { - if (err) { - log.trace('error processing request', { error: err, method: 'bucketPutLogging' }); - monitoring.promMetrics('PUT', bucketName, err.code, 'bucketPutLogging'); - return callback(err); - } - - monitoring.promMetrics('PUT', bucketName, '200', 'bucketPutLogging'); - return callback(); - }); + ); } module.exports = bucketPutLogging; diff --git a/lib/api/bucketPutNotification.js b/lib/api/bucketPutNotification.js index ab82b5109a..4719f88fab 100644 --- a/lib/api/bucketPutNotification.js +++ b/lib/api/bucketPutNotification.js @@ -27,34 +27,36 @@ function bucketPutNotification(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => parseXML(request.post, log, next), - (parsedXml, next) => { - const notificationConfig = getNotificationConfiguration(parsedXml); - const notifConfig = notificationConfig.error ? undefined : notificationConfig; - process.nextTick(() => next(notificationConfig.error, notifConfig)); + return async.waterfall( + [ + next => parseXML(request.post, log, next), + (parsedXml, next) => { + const notificationConfig = getNotificationConfiguration(parsedXml); + const notifConfig = notificationConfig.error ? undefined : notificationConfig; + process.nextTick(() => next(notificationConfig.error, notifConfig)); + }, + (notifConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket, notifConfig), + ), + (bucket, notifConfig, next) => { + bucket.setNotificationConfiguration(notifConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutNotification' }); + return callback(err, corsHeaders); + } + pushMetric('putBucketNotification', log, { + authInfo, + bucket: bucketName, + }); + return callback(null, corsHeaders); }, - (notifConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket, notifConfig)), - (bucket, notifConfig, next) => { - bucket.setNotificationConfiguration(notifConfig); - metadata.updateBucket(bucket.getName(), bucket, log, - err => next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutNotification' }); - return callback(err, corsHeaders); - } - pushMetric('putBucketNotification', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutNotification; diff --git a/lib/api/bucketPutObjectLock.js b/lib/api/bucketPutObjectLock.js index a3d549789a..cb7b20bd63 100644 --- a/lib/api/bucketPutObjectLock.js +++ b/lib/api/bucketPutObjectLock.js @@ -2,7 +2,9 @@ const { waterfall } = require('async'); const arsenal = require('arsenal'); const { errorInstances } = arsenal; -const { models: { ObjectLockConfiguration } } = arsenal; +const { + models: { ObjectLockConfiguration }, +} = arsenal; const parseXML = require('../utilities/parseXML'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); @@ -29,55 +31,57 @@ function bucketPutObjectLock(authInfo, request, log, callback) { requestType: request.apiMethods || 'bucketPutObjectLock', request, }; - return waterfall([ - next => parseXML(request.post, log, next), - (parsedXml, next) => { - const lockConfigClass = new ObjectLockConfiguration(parsedXml); - // if there was an error getting object lock configuration, - // returned configObj will contain 'error' key - process.nextTick(() => { - const configObj = lockConfigClass. - getValidatedObjectLockConfiguration(); - return next(configObj.error || null, configObj); + return waterfall( + [ + next => parseXML(request.post, log, next), + (parsedXml, next) => { + const lockConfigClass = new ObjectLockConfiguration(parsedXml); + // if there was an error getting object lock configuration, + // returned configObj will contain 'error' key + process.nextTick(() => { + const configObj = lockConfigClass.getValidatedObjectLockConfiguration(); + return next(configObj.error || null, configObj); + }); + }, + (objectLockConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, objectLockConfig); + }), + (bucket, objectLockConfig, next) => { + const isObjectLockEnabled = bucket.isObjectLockEnabled(); + process.nextTick(() => { + if (!isObjectLockEnabled) { + return next( + errorInstances.InvalidBucketState.customizeDescription( + 'Object Lock configuration cannot be enabled on ' + 'existing buckets', + ), + bucket, + ); + } + return next(null, bucket, objectLockConfig); + }); + }, + (bucket, objectLockConfig, next) => { + bucket.setObjectLockConfiguration(objectLockConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutObjectLock' }); + return callback(err, corsHeaders); + } + pushMetric('putBucketObjectLock', log, { + authInfo, + bucket: bucketName, }); + return callback(null, corsHeaders); }, - (objectLockConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, - log, (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, objectLockConfig); - }), - (bucket, objectLockConfig, next) => { - const isObjectLockEnabled = bucket.isObjectLockEnabled(); - process.nextTick(() => { - if (!isObjectLockEnabled) { - return next(errorInstances.InvalidBucketState.customizeDescription( - 'Object Lock configuration cannot be enabled on ' + - 'existing buckets'), bucket); - } - return next(null, bucket, objectLockConfig); - }); - }, - (bucket, objectLockConfig, next) => { - bucket.setObjectLockConfiguration(objectLockConfig); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutObjectLock' }); - return callback(err, corsHeaders); - } - pushMetric('putBucketObjectLock', log, { - authInfo, - bucket: bucketName, - }); - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutObjectLock; diff --git a/lib/api/bucketPutPolicy.js b/lib/api/bucketPutPolicy.js index 56f48e2a7f..157171303b 100644 --- a/lib/api/bucketPutPolicy.js +++ b/lib/api/bucketPutPolicy.js @@ -3,8 +3,7 @@ const { errorInstances, models } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); -const { validatePolicyResource, validatePolicyConditions } = - require('./apiUtils/authorization/permissionChecks'); +const { validatePolicyResource, validatePolicyConditions } = require('./apiUtils/authorization/permissionChecks'); const { BucketPolicy } = models; /** @@ -16,8 +15,7 @@ const { BucketPolicy } = models; function _checkNotImplementedPolicy(policyString) { // bucket names and key names cannot include "", so including those // isolates not implemented keys - return policyString.includes('"Service"') - || policyString.includes('"Federated"'); + return policyString.includes('"Service"') || policyString.includes('"Federated"'); } /** @@ -39,58 +37,57 @@ function bucketPutPolicy(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => { - const bucketPolicy = new BucketPolicy(request.post); - // if there was an error getting bucket policy, - // returned policyObj will contain 'error' key - process.nextTick(() => { - const policyObj = bucketPolicy.getBucketPolicy(); - if (_checkNotImplementedPolicy(request.post)) { - const err = errorInstances.NotImplemented.customizeDescription( - 'Bucket policy contains element not yet implemented'); - return next(err); - } - if (policyObj.error) { - const err = errorInstances.MalformedPolicy.customizeDescription( - policyObj.error.description); - return next(err); - } - return next(null, policyObj); - }); + return async.waterfall( + [ + next => { + const bucketPolicy = new BucketPolicy(request.post); + // if there was an error getting bucket policy, + // returned policyObj will contain 'error' key + process.nextTick(() => { + const policyObj = bucketPolicy.getBucketPolicy(); + if (_checkNotImplementedPolicy(request.post)) { + const err = errorInstances.NotImplemented.customizeDescription( + 'Bucket policy contains element not yet implemented', + ); + return next(err); + } + if (policyObj.error) { + const err = errorInstances.MalformedPolicy.customizeDescription(policyObj.error.description); + return next(err); + } + return next(null, policyObj); + }); + }, + (bucketPolicy, next) => { + process.nextTick(() => { + if (!validatePolicyResource(bucketName, bucketPolicy)) { + return next(errorInstances.MalformedPolicy.customizeDescription('Policy has invalid resource')); + } + return next(validatePolicyConditions(bucketPolicy), bucketPolicy); + }); + }, + (bucketPolicy, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, bucketPolicy); + }), + (bucket, bucketPolicy, next) => { + bucket.setBucketPolicy(bucketPolicy); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutPolicy' }); + return callback(err, corsHeaders); + } + // TODO: implement Utapi metric support + return callback(null, corsHeaders); }, - (bucketPolicy, next) => { - process.nextTick(() => { - if (!validatePolicyResource(bucketName, bucketPolicy)) { - return next(errorInstances.MalformedPolicy.customizeDescription( - 'Policy has invalid resource')); - } - return next(validatePolicyConditions(bucketPolicy), bucketPolicy); - }); - }, - (bucketPolicy, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, bucketPolicy); - }), - (bucket, bucketPolicy, next) => { - bucket.setBucketPolicy(bucketPolicy); - metadata.updateBucket(bucket.getName(), bucket, log, - err => next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'bucketPutPolicy' }); - return callback(err, corsHeaders); - } - // TODO: implement Utapi metric support - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutPolicy; diff --git a/lib/api/bucketPutRateLimit.js b/lib/api/bucketPutRateLimit.js index d895678775..e10b06c2e1 100644 --- a/lib/api/bucketPutRateLimit.js +++ b/lib/api/bucketPutRateLimit.js @@ -20,8 +20,9 @@ function parseRequestBody(requestBody, callback) { function validateRateLimitConfig(requestConfig, callback) { const limit = requestConfig.RequestsPerSecond; if (Number.isNaN(limit) || !Number.isInteger(limit) || limit < 0) { - return callback(errorInstances.InvalidArgument - .customizeDescription('RequestsPerSecond must be a positive integer')); + return callback( + errorInstances.InvalidArgument.customizeDescription('RequestsPerSecond must be a positive integer'), + ); } // Validate limit against node count AND worker count @@ -30,21 +31,25 @@ function validateRateLimitConfig(requestConfig, callback) { const minLimit = nodes * workers; if (limit > 0 && limit < minLimit) { - return callback(errorInstances.InvalidArgument - .customizeDescription( + return callback( + errorInstances.InvalidArgument.customizeDescription( `RequestsPerSecond (${limit}) must be >= ` + - `(nodes x workers = ${nodes} x ${workers} = ${minLimit}) or 0 (unlimited). ` + - 'Each worker enforces limit/nodes/workers locally. ' + - `With limit less than ${minLimit}, per-worker rate would be less than 1 req/s, ` + - 'effectively blocking traffic.' - )); + `(nodes x workers = ${nodes} x ${workers} = ${minLimit}) or 0 (unlimited). ` + + 'Each worker enforces limit/nodes/workers locally. ' + + `With limit less than ${minLimit}, per-worker rate would be less than 1 req/s, ` + + 'effectively blocking traffic.', + ), + ); } - return callback(null, new models.RateLimitConfiguration({ - RequestsPerSecond: { - Limit: limit, - }, - })); + return callback( + null, + new models.RateLimitConfiguration({ + RequestsPerSecond: { + Limit: limit, + }, + }), + ); } /** @@ -70,35 +75,35 @@ function bucketPutRateLimit(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => parseRequestBody(request.post, next), - (requestBody, next) => validateRateLimitConfig(requestBody, next), - (limitConfig, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - if (err) { - return next(err, bucket); - } - return next(null, bucket, limitConfig); - }), - (bucket, limitConfig, next) => { - bucket.setRateLimitConfiguration(limitConfig); - metadata.updateBucket(bucket.getName(), bucket, log, - err => next(err, bucket)); + return async.waterfall( + [ + next => parseRequestBody(request.post, next), + (requestBody, next) => validateRateLimitConfig(requestBody, next), + (limitConfig, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err, bucket); + } + return next(null, bucket, limitConfig); + }), + (bucket, limitConfig, next) => { + bucket.setRateLimitConfiguration(limitConfig); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutRateLimit' }); + return callback(err, corsHeaders); + } + // Invalidate cache so new limit takes effect immediately + cache.deleteCachedConfig(cache.namespace.bucket, bucketName); + log.debug('invalidated rate limit cache for bucket', { bucketName }); + // TODO: implement Utapi metric support + return callback(null, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'bucketPutRateLimit' }); - return callback(err, corsHeaders); - } - // Invalidate cache so new limit takes effect immediately - cache.deleteCachedConfig(cache.namespace.bucket, bucketName); - log.debug('invalidated rate limit cache for bucket', { bucketName }); - // TODO: implement Utapi metric support - return callback(null, corsHeaders); - }); + ); } module.exports = bucketPutRateLimit; diff --git a/lib/api/bucketPutReplication.js b/lib/api/bucketPutReplication.js index c940e660b4..a5a62794a0 100644 --- a/lib/api/bucketPutReplication.js +++ b/lib/api/bucketPutReplication.js @@ -4,17 +4,15 @@ const { errorInstances } = require('arsenal'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); -const { getReplicationConfiguration } = - require('./apiUtils/bucket/getReplicationConfiguration'); -const validateConfiguration = - require('./apiUtils/bucket/validateReplicationConfig'); +const { getReplicationConfiguration } = require('./apiUtils/bucket/getReplicationConfiguration'); +const validateConfiguration = require('./apiUtils/bucket/validateReplicationConfig'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const monitoring = require('../utilities/monitoringHandler'); // The error response when a bucket does not have versioning 'Enabled'. const versioningNotEnabledError = errorInstances.InvalidRequest.customizeDescription( - 'Versioning must be \'Enabled\' on the bucket to apply a replication ' + - 'configuration'); + "Versioning must be 'Enabled' on the bucket to apply a replication " + 'configuration', +); /** * bucketPutReplication - Create or update bucket replication configuration @@ -34,57 +32,55 @@ function bucketPutReplication(authInfo, request, log, callback) { request, }; - return waterfall([ - // Validate the request XML and return the replication configuration. - next => getReplicationConfiguration(post, log, next), - // Check bucket user privileges and ensure versioning is 'Enabled'. - (config, next) => - // TODO: Validate that destination bucket exists and has versioning. - standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - if (err) { - return next(err); + return waterfall( + [ + // Validate the request XML and return the replication configuration. + next => getReplicationConfiguration(post, log, next), + // Check bucket user privileges and ensure versioning is 'Enabled'. + (config, next) => + // TODO: Validate that destination bucket exists and has versioning. + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + if (err) { + return next(err); + } + // Replication requires that versioning is 'Enabled' unless it + // is an NFS bucket. + if (!bucket.isNFS() && !bucket.isVersioningEnabled(bucket)) { + return next(versioningNotEnabledError); + } + return next(null, config, bucket); + }), + // Set the replication configuration and update the bucket metadata. + (config, bucket, next) => { + // validate there's a preferred read location in case the + // bucket location is a transient source + if (!validateConfiguration(config, bucket)) { + const msg = 'Replication configuration lacks a preferred ' + 'read location'; + log.error(msg, { bucketName: bucket.getName() }); + return next(errorInstances.ValidationError.customizeDescription(msg)); } - // Replication requires that versioning is 'Enabled' unless it - // is an NFS bucket. - if (!bucket.isNFS() && !bucket.isVersioningEnabled(bucket)) { - return next(versioningNotEnabledError); - } - return next(null, config, bucket); - }), - // Set the replication configuration and update the bucket metadata. - (config, bucket, next) => { - // validate there's a preferred read location in case the - // bucket location is a transient source - if (!validateConfiguration(config, bucket)) { - const msg = 'Replication configuration lacks a preferred ' + - 'read location'; - log.error(msg, { bucketName: bucket.getName() }); - return next(errorInstances.ValidationError - .customizeDescription(msg)); + bucket.setReplicationConfiguration(config); + return metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(headers.origin, method, bucket); + if (err) { + log.trace('error processing request', { + error: err, + method: 'bucketPutReplication', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketReplication'); + return callback(err, corsHeaders); } - bucket.setReplicationConfiguration(config); - return metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); - }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(headers.origin, method, bucket); - if (err) { - log.trace('error processing request', { - error: err, - method: 'bucketPutReplication', + pushMetric('putBucketReplication', log, { + authInfo, + bucket: bucketName, }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putBucketReplication'); - return callback(err, corsHeaders); - } - pushMetric('putBucketReplication', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putBucketReplication'); - return callback(null, corsHeaders); - }); + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketReplication'); + return callback(null, corsHeaders); + }, + ); } module.exports = bucketPutReplication; diff --git a/lib/api/bucketPutTagging.js b/lib/api/bucketPutTagging.js index 9023f48504..292133d223 100644 --- a/lib/api/bucketPutTagging.js +++ b/lib/api/bucketPutTagging.js @@ -1,7 +1,6 @@ const { waterfall } = require('async'); const { s3middleware } = require('arsenal'); - const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const metadata = require('../metadata/wrapper'); @@ -42,43 +41,42 @@ function bucketPutTagging(authInfo, request, log, callback) { request, }; let bucket = null; - return waterfall([ - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, b) => { - bucket = b; - return next(err); - }), - next => checkExpectedBucketOwner(headers, bucket, log, next), - next => parseTagXml(request.post, log, next), - (tags, next) => { - const tagArray = []; - Object.keys(tags).forEach(key => { - tagArray.push({ Value: tags[key], Key: key }); - }); - bucket.setTags(tagArray); - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err)); + return waterfall( + [ + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, b) => { + bucket = b; + return next(err); + }), + next => checkExpectedBucketOwner(headers, bucket, log, next), + next => parseTagXml(request.post, log, next), + (tags, next) => { + const tagArray = []; + Object.keys(tags).forEach(key => { + tagArray.push({ Value: tags[key], Key: key }); + }); + bucket.setTags(tagArray); + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err)); + }, + ], + err => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, + method: 'bucketPutTagging', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketTagging'); + } else { + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketTagging'); + pushMetric('putBucketTagging', log, { + authInfo, + bucket: bucketName, + }); + } + return callback(err, corsHeaders); }, - ], err => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'bucketPutTagging' - }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putBucketTagging'); - } else { - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putBucketTagging'); - pushMetric('putBucketTagging', log, { - authInfo, - bucket: bucketName, - }); - } - return callback(err, corsHeaders); - }); + ); } module.exports = bucketPutTagging; diff --git a/lib/api/bucketPutVersioning.js b/lib/api/bucketPutVersioning.js index 5f872cc0dd..cb5c6dd07b 100644 --- a/lib/api/bucketPutVersioning.js +++ b/lib/api/bucketPutVersioning.js @@ -6,23 +6,23 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucket } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); -const versioningNotImplBackends = - require('../../constants').versioningNotImplBackends; +const versioningNotImplBackends = require('../../constants').versioningNotImplBackends; const { config } = require('../Config'); const monitoring = require('../utilities/monitoringHandler'); -const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; -const replicationVersioningErrorMessage = 'A replication configuration is ' + -'present on this bucket, so you cannot change the versioning state. To ' + -'change the versioning state, first delete the replication configuration.'; +const replicationVersioningErrorMessage = + 'A replication configuration is ' + + 'present on this bucket, so you cannot change the versioning state. To ' + + 'change the versioning state, first delete the replication configuration.'; -const ingestionVersioningErrorMessage = 'Versioning cannot be suspended for ' -+ 'buckets setup with Out of Band updates from a location'; +const ingestionVersioningErrorMessage = + 'Versioning cannot be suspended for ' + 'buckets setup with Out of Band updates from a location'; -const objectLockErrorMessage = 'An Object Lock configuration is present on ' + - 'this bucket, so the versioning state cannot be changed.'; +const objectLockErrorMessage = + 'An Object Lock configuration is present on ' + 'this bucket, so the versioning state cannot be changed.'; /** * Format of xml request: @@ -47,21 +47,17 @@ function _parseXML(request, log, cb) { return cb(errors.MalformedXML); } const versioningConf = result.VersioningConfiguration; - const status = versioningConf.Status ? - versioningConf.Status[0] : undefined; - const mfaDelete = versioningConf.MfaDelete ? - versioningConf.MfaDelete[0] : undefined; + const status = versioningConf.Status ? versioningConf.Status[0] : undefined; + const mfaDelete = versioningConf.MfaDelete ? versioningConf.MfaDelete[0] : undefined; const validStatuses = ['Enabled', 'Suspended']; const validMfaDeletes = [undefined, 'Enabled', 'Disabled']; - if (validStatuses.indexOf(status) < 0 || - validMfaDeletes.indexOf(mfaDelete) < 0) { + if (validStatuses.indexOf(status) < 0 || validMfaDeletes.indexOf(mfaDelete) < 0) { log.debug('illegal versioning configuration'); return cb(errors.IllegalVersioningConfigurationException); } if (versioningConf && mfaDelete === 'Enabled') { log.debug('mfa deletion is not implemented'); - return cb(errorInstances.NotImplemented - .customizeDescription('MFA Deletion is not supported yet.')); + return cb(errorInstances.NotImplemented.customizeDescription('MFA Deletion is not supported yet.')); } return process.nextTick(() => cb(null)); }); @@ -103,90 +99,89 @@ function bucketPutVersioning(authInfo, request, log, callback) { requestType: request.apiMethods || 'bucketPutVersioning', request, }; - return waterfall([ - next => _parseXML(request, log, next), - next => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket)), // ignore extra null object, - (bucket, next) => parseString(request.post, (err, result) => { - // just for linting; there should not be any parsing error here - if (err) { - return next(err, bucket); - } - // prevent enabling versioning on an nfs exported bucket - if (bucket.isNFS()) { - const error = new Error(); - error.code = 'NFSBUCKET'; - return next(error); - } - // _checkBackendVersioningImplemented returns false if versioning - // is not implemented on the bucket backend - if (!_checkBackendVersioningImplemented(bucket)) { - log.debug(externalVersioningErrorMessage, - { method: 'bucketPutVersioning', - error: errors.NotImplemented }); - const error = errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage); - return next(error, bucket); - } - const versioningConfiguration = {}; - if (result.VersioningConfiguration.Status) { - versioningConfiguration.Status = - result.VersioningConfiguration.Status[0]; - } - if (result.VersioningConfiguration.MfaDelete) { - versioningConfiguration.MfaDelete = - result.VersioningConfiguration.MfaDelete[0]; - } - // the configuration has been checked before - return next(null, bucket, versioningConfiguration); - }), - (bucket, versioningConfiguration, next) => { - // check if replication is enabled if versioning is being suspended - const replicationConfig = bucket.getReplicationConfiguration(); - const isIngestionBucket = bucket.isIngestionBucket && bucket.isIngestionBucket(); - const invalidAction = - versioningConfiguration.Status === 'Suspended' - && (isIngestionBucket || replicationConfig?.rules?.some(r => r.enabled)); - if (invalidAction) { - const errorMsg = isIngestionBucket ? - ingestionVersioningErrorMessage : replicationVersioningErrorMessage; - next(errorInstances.InvalidBucketState - .customizeDescription(errorMsg)); - return; + return waterfall( + [ + next => _parseXML(request, log, next), + next => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket), + ), // ignore extra null object, + (bucket, next) => + parseString(request.post, (err, result) => { + // just for linting; there should not be any parsing error here + if (err) { + return next(err, bucket); + } + // prevent enabling versioning on an nfs exported bucket + if (bucket.isNFS()) { + const error = new Error(); + error.code = 'NFSBUCKET'; + return next(error); + } + // _checkBackendVersioningImplemented returns false if versioning + // is not implemented on the bucket backend + if (!_checkBackendVersioningImplemented(bucket)) { + log.debug(externalVersioningErrorMessage, { + method: 'bucketPutVersioning', + error: errors.NotImplemented, + }); + const error = + errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage); + return next(error, bucket); + } + const versioningConfiguration = {}; + if (result.VersioningConfiguration.Status) { + versioningConfiguration.Status = result.VersioningConfiguration.Status[0]; + } + if (result.VersioningConfiguration.MfaDelete) { + versioningConfiguration.MfaDelete = result.VersioningConfiguration.MfaDelete[0]; + } + // the configuration has been checked before + return next(null, bucket, versioningConfiguration); + }), + (bucket, versioningConfiguration, next) => { + // check if replication is enabled if versioning is being suspended + const replicationConfig = bucket.getReplicationConfiguration(); + const isIngestionBucket = bucket.isIngestionBucket && bucket.isIngestionBucket(); + const invalidAction = + versioningConfiguration.Status === 'Suspended' && + (isIngestionBucket || replicationConfig?.rules?.some(r => r.enabled)); + if (invalidAction) { + const errorMsg = isIngestionBucket + ? ingestionVersioningErrorMessage + : replicationVersioningErrorMessage; + next(errorInstances.InvalidBucketState.customizeDescription(errorMsg)); + return; + } + const objectLockEnabled = bucket.isObjectLockEnabled(); + if (objectLockEnabled) { + next(errorInstances.InvalidBucketState.customizeDescription(objectLockErrorMessage)); + return; + } + bucket.setVersioningConfiguration(versioningConfiguration); + // TODO all metadata updates of bucket should be using CAS + metadata.updateBucket(bucket.getName(), bucket, log, err => next(err, bucket)); + }, + ], + (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err && err.code === 'NFSBUCKET') { + log.trace('skipping versioning for nfs exported bucket'); + return callback(null, corsHeaders); } - const objectLockEnabled = bucket.isObjectLockEnabled(); - if (objectLockEnabled) { - next(errorInstances.InvalidBucketState - .customizeDescription(objectLockErrorMessage)); - return; + if (err) { + log.trace('error processing request', { error: err, method: 'bucketPutVersioning' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putBucketVersioning'); + } else { + pushMetric('putBucketVersioning', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putBucketVersioning'); } - bucket.setVersioningConfiguration(versioningConfiguration); - // TODO all metadata updates of bucket should be using CAS - metadata.updateBucket(bucket.getName(), bucket, log, err => - next(err, bucket)); + return callback(err, corsHeaders); }, - ], (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err && err.code === 'NFSBUCKET') { - log.trace('skipping versioning for nfs exported bucket'); - return callback(null, corsHeaders); - } - if (err) { - log.trace('error processing request', { error: err, - method: 'bucketPutVersioning' }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putBucketVersioning'); - } else { - pushMetric('putBucketVersioning', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putBucketVersioning'); - } - return callback(err, corsHeaders); - }); + ); } module.exports = bucketPutVersioning; diff --git a/lib/api/bucketPutWebsite.js b/lib/api/bucketPutWebsite.js index b7110a8a11..7fd1374d6c 100644 --- a/lib/api/bucketPutWebsite.js +++ b/lib/api/bucketPutWebsite.js @@ -34,42 +34,45 @@ function bucketPutWebsite(authInfo, request, log, callback) { return callback(errors.MissingRequestBodyError); } - return async.waterfall([ - next => { - log.trace('parsing website configuration'); - return parseWebsiteConfigXml(request.post, log, next); - }, - (config, next) => standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); - if (err) { - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - if (err?.is?.AccessDenied) { - return next(err, corsHeaders); + return async.waterfall( + [ + next => { + log.trace('parsing website configuration'); + return parseWebsiteConfigXml(request.post, log, next); + }, + (config, next) => + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + if (err?.is?.AccessDenied) { + return next(err, corsHeaders); + } + return next(err); } - return next(err); - } - return next(null, bucket, config, corsHeaders); - }), - (bucket, config, corsHeaders, next) => { - log.trace('updating bucket website configuration in metadata'); - bucket.setWebsiteConfiguration(config); - return metadata.updateBucket(bucketName, bucket, log, err => { - next(err, corsHeaders); - }); - } - ], (err, corsHeaders) => { - if (err) { - log.trace('error processing request', { error: err, method: REQUEST_TYPE }); - monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); - return callback(err, corsHeaders); - } + return next(null, bucket, config, corsHeaders); + }), + (bucket, config, corsHeaders, next) => { + log.trace('updating bucket website configuration in metadata'); + bucket.setWebsiteConfiguration(config); + return metadata.updateBucket(bucketName, bucket, log, err => { + next(err, corsHeaders); + }); + }, + ], + (err, corsHeaders) => { + if (err) { + log.trace('error processing request', { error: err, method: REQUEST_TYPE }); + monitoring.promMetrics('PUT', bucketName, err.code, METRICS_ACTION); + return callback(err, corsHeaders); + } - pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); - monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); - return callback(null, corsHeaders); - }); + pushMetric(METRICS_ACTION, log, { authInfo, bucket: bucketName }); + monitoring.promMetrics('PUT', bucketName, '200', METRICS_ACTION); + return callback(null, corsHeaders); + }, + ); } module.exports = bucketPutWebsite; diff --git a/lib/api/corsPreflight.js b/lib/api/corsPreflight.js index 78ad92caf5..0200b019ea 100644 --- a/lib/api/corsPreflight.js +++ b/lib/api/corsPreflight.js @@ -2,42 +2,45 @@ const { errors, errorInstances } = require('arsenal'); const metadata = require('../metadata/wrapper'); const bucketShield = require('./apiUtils/bucket/bucketShield'); -const { findCorsRule, generateCorsResHeaders } - = require('./apiUtils/object/corsResponse'); +const { findCorsRule, generateCorsResHeaders } = require('./apiUtils/object/corsResponse'); // const { pushMetric } = require('../utapi/utilities'); const requestType = 'objectGet'; const customizedErrs = { corsNotEnabled: 'CORSResponse: CORS is not enabled for this bucket.', - notAllowed: 'CORSResponse: This CORS request is not allowed. ' + - 'This is usually because the evalution of Origin, request method / ' + - 'Access-Control-Request-Method or Access-Control-Request-Headers ' + - 'are not whitelisted by the resource\'s CORS spec.', + notAllowed: + 'CORSResponse: This CORS request is not allowed. ' + + 'This is usually because the evalution of Origin, request method / ' + + 'Access-Control-Request-Method or Access-Control-Request-Headers ' + + "are not whitelisted by the resource's CORS spec.", }; /** corsPreflight - handle preflight CORS requests -* @param {object} request - http request object -* @param {function} log - Werelogs request logger -* @param {function} callback - callback to respond to http request -* with either error code or 200 response -* @return {undefined} -*/ + * @param {object} request - http request object + * @param {function} log - Werelogs request logger + * @param {function} callback - callback to respond to http request + * with either error code or 200 response + * @return {undefined} + */ function corsPreflight(request, log, callback) { log.debug('processing request', { method: 'corsPreflight' }); const bucketName = request.bucketName; const corsOrigin = request.headers.origin; const corsMethod = request.headers['access-control-request-method']; - const corsHeaders = request.headers['access-control-request-headers'] ? - request.headers['access-control-request-headers'].replace(/ /g, '') - .split(',').reduce((resultArr, value) => { - // remove empty values and convert values to lowercase - if (value !== '') { - resultArr.push(value.toLowerCase()); - } - return resultArr; - }, []) : null; + const corsHeaders = request.headers['access-control-request-headers'] + ? request.headers['access-control-request-headers'] + .replace(/ /g, '') + .split(',') + .reduce((resultArr, value) => { + // remove empty values and convert values to lowercase + if (value !== '') { + resultArr.push(value.toLowerCase()); + } + return resultArr; + }, []) + : null; return metadata.getBucket(bucketName, log, (err, bucket) => { if (err) { @@ -51,8 +54,7 @@ function corsPreflight(request, log, callback) { const corsRules = bucket.getCors(); if (!corsRules) { - const err = errorInstances.AccessForbidden - .customizeDescription(customizedErrs.corsNotEnabled); + const err = errorInstances.AccessForbidden.customizeDescription(customizedErrs.corsNotEnabled); log.trace('no existing cors configuration', { error: err, method: 'corsPreflight', @@ -61,12 +63,10 @@ function corsPreflight(request, log, callback) { } log.trace('finding cors rule'); - const corsRule = findCorsRule(corsRules, corsOrigin, corsMethod, - corsHeaders); + const corsRule = findCorsRule(corsRules, corsOrigin, corsMethod, corsHeaders); if (!corsRule) { - const err = errorInstances.AccessForbidden - .customizeDescription(customizedErrs.notAllowed); + const err = errorInstances.AccessForbidden.customizeDescription(customizedErrs.notAllowed); log.trace('no matching cors rule', { error: err, method: 'corsPreflight', @@ -74,8 +74,7 @@ function corsPreflight(request, log, callback) { return callback(err); } - const resHeaders = generateCorsResHeaders(corsRule, corsOrigin, - corsMethod, corsHeaders, true); + const resHeaders = generateCorsResHeaders(corsRule, corsOrigin, corsMethod, corsHeaders, true); // TODO: add some level of metrics for non-standard API request: // pushMetric('corsPreflight', log, { bucket: bucketName }); return callback(null, resHeaders); diff --git a/lib/api/listMultipartUploads.js b/lib/api/listMultipartUploads.js index 4a93b0439c..ad18d783c0 100644 --- a/lib/api/listMultipartUploads.js +++ b/lib/api/listMultipartUploads.js @@ -103,72 +103,71 @@ function listMultipartUploads(authInfo, request, log, callback) { request, }; - async.waterfall([ - function waterfall1(next) { - // Check final destination bucket for authorization rather - // than multipart upload bucket - standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, - (err, bucket) => next(err, bucket)); - }, - function getMPUBucket(bucket, next) { - services.getMPUBucket(bucket, bucketName, log, - (err, mpuBucket) => next(err, bucket, mpuBucket)); - }, - function waterfall2(bucket, mpuBucket, next) { - let splitter = constants.splitter; - // BACKWARD: Remove to remove the old splitter - if (mpuBucket.getMdBucketModelVersion() < 2) { - splitter = constants.oldSplitter; - } - let maxUploads = query['max-uploads'] !== undefined ? - Number.parseInt(query['max-uploads'], 10) : 1000; - if (maxUploads < 0) { - monitoring.promMetrics('GET', bucketName, 400, - 'listMultipartUploads'); - return callback(errors.InvalidArgument, bucket); - } - if (maxUploads > constants.listingHardLimit) { - maxUploads = constants.listingHardLimit; + async.waterfall( + [ + function waterfall1(next) { + // Check final destination bucket for authorization rather + // than multipart upload bucket + standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => + next(err, bucket), + ); + }, + function getMPUBucket(bucket, next) { + services.getMPUBucket(bucket, bucketName, log, (err, mpuBucket) => next(err, bucket, mpuBucket)); + }, + function waterfall2(bucket, mpuBucket, next) { + let splitter = constants.splitter; + // BACKWARD: Remove to remove the old splitter + if (mpuBucket.getMdBucketModelVersion() < 2) { + splitter = constants.oldSplitter; + } + let maxUploads = query['max-uploads'] !== undefined ? Number.parseInt(query['max-uploads'], 10) : 1000; + if (maxUploads < 0) { + monitoring.promMetrics('GET', bucketName, 400, 'listMultipartUploads'); + return callback(errors.InvalidArgument, bucket); + } + if (maxUploads > constants.listingHardLimit) { + maxUploads = constants.listingHardLimit; + } + const listingParams = { + delimiter: query.delimiter, + keyMarker: query['key-marker'], + uploadIdMarker: query['upload-id-marker'], + maxKeys: maxUploads, + prefix: `overview${splitter}${prefix}`, + queryPrefixLength: prefix.length, + listingType: 'MPU', + splitter, + }; + services.getMultipartUploadListing(mpuBucketName, listingParams, log, (err, list) => + next(err, bucket, list), + ); + return undefined; + }, + ], + (err, bucket, list) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('GET', bucketName, err.code, 'listMultipartUploads'); + return callback(err, null, corsHeaders); } - const listingParams = { - delimiter: query.delimiter, + const xmlParams = { + bucketName, + encoding, + list, + prefix: query.prefix, keyMarker: query['key-marker'], uploadIdMarker: query['upload-id-marker'], - maxKeys: maxUploads, - prefix: `overview${splitter}${prefix}`, - queryPrefixLength: prefix.length, - listingType: 'MPU', - splitter, }; - services.getMultipartUploadListing(mpuBucketName, listingParams, - log, (err, list) => next(err, bucket, list)); - return undefined; + const xml = convertToXml('listMultipartUploads', xmlParams); + pushMetric('listMultipartUploads', log, { + authInfo, + bucket: bucketName, + }); + monitoring.promMetrics('GET', bucketName, '200', 'listMultipartUploads'); + return callback(null, xml, corsHeaders); }, - ], (err, bucket, list) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - monitoring.promMetrics('GET', bucketName, err.code, - 'listMultipartUploads'); - return callback(err, null, corsHeaders); - } - const xmlParams = { - bucketName, - encoding, - list, - prefix: query.prefix, - keyMarker: query['key-marker'], - uploadIdMarker: query['upload-id-marker'], - }; - const xml = convertToXml('listMultipartUploads', xmlParams); - pushMetric('listMultipartUploads', log, { - authInfo, - bucket: bucketName, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'listMultipartUploads'); - return callback(null, xml, corsHeaders); - }); + ); } module.exports = listMultipartUploads; diff --git a/lib/api/metadataSearch.js b/lib/api/metadataSearch.js index ba4f355abc..382472eb08 100644 --- a/lib/api/metadataSearch.js +++ b/lib/api/metadataSearch.js @@ -8,13 +8,10 @@ const validateSearchParams = require('../api/apiUtils/bucket/validateSearch'); const parseWhere = require('../api/apiUtils/bucket/parseWhere'); const versionIdUtils = versioning.VersionID; const monitoring = require('../utilities/monitoringHandler'); -const { decryptToken } - = require('../api/apiUtils/object/continueToken'); +const { decryptToken } = require('../api/apiUtils/object/continueToken'); const { processVersions, processMasterVersions } = require('./bucketGet'); - -function handleResult(listParams, requestMaxKeys, encoding, authInfo, - bucketName, list, corsHeaders, log, callback) { +function handleResult(listParams, requestMaxKeys, encoding, authInfo, bucketName, list, corsHeaders, log, callback) { // eslint-disable-next-line no-param-reassign listParams.maxKeys = requestMaxKeys; // eslint-disable-next-line no-param-reassign @@ -47,22 +44,21 @@ function metadataSearch(authInfo, request, log, callback) { const bucketName = request.bucketName; const v2 = params['list-type']; if (v2 !== undefined && Number.parseInt(v2, 10) !== 2) { - return callback(errorInstances.InvalidArgument.customizeDescription('Invalid ' + - 'List Type specified in Request')); + return callback( + errorInstances.InvalidArgument.customizeDescription('Invalid ' + 'List Type specified in Request'), + ); } log.debug('processing request', { method: 'metadataSearch' }); const encoding = params['encoding-type']; if (encoding !== undefined && encoding !== 'url') { - monitoring.promMetrics( - 'GET', bucketName, 400, 'metadataSearch'); - return callback(errorInstances.InvalidArgument.customizeDescription('Invalid ' + - 'Encoding Method specified in Request')); + monitoring.promMetrics('GET', bucketName, 400, 'metadataSearch'); + return callback( + errorInstances.InvalidArgument.customizeDescription('Invalid ' + 'Encoding Method specified in Request'), + ); } - const requestMaxKeys = params['max-keys'] ? - Number.parseInt(params['max-keys'], 10) : 1000; + const requestMaxKeys = params['max-keys'] ? Number.parseInt(params['max-keys'], 10) : 1000; if (Number.isNaN(requestMaxKeys) || requestMaxKeys < 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'metadataSearch'); + monitoring.promMetrics('GET', bucketName, 400, 'metadataSearch'); return callback(errors.InvalidArgument); } // AWS only returns 1000 keys even if max keys are greater. @@ -89,37 +85,34 @@ function metadataSearch(authInfo, request, log, callback) { log.debug(err.message, { stack: err.stack, }); - monitoring.promMetrics( - 'GET', bucketName, 400, 'metadataSearch'); - return callback(errorInstances.InvalidArgument - .customizeDescription('Invalid sql where clause ' + - 'sent as search query')); + monitoring.promMetrics('GET', bucketName, 400, 'metadataSearch'); + return callback( + errorInstances.InvalidArgument.customizeDescription('Invalid sql where clause ' + 'sent as search query'), + ); } if (v2) { listParams.v2 = true; listParams.startAfter = params['start-after']; - listParams.continuationToken = - decryptToken(params['continuation-token']); + listParams.continuationToken = decryptToken(params['continuation-token']); listParams.fetchOwner = params['fetch-owner'] === 'true'; } else { listParams.marker = params.marker; } standardMetadataValidateBucket(metadataValParams, request.actionImplicitDenies, log, (err, bucket) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { log.debug('error processing request', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'metadataSearch'); + monitoring.promMetrics('GET', bucketName, err.code, 'metadataSearch'); return callback(err, null, corsHeaders); } if (params.versions !== undefined) { listParams.listingType = 'DelimiterVersions'; delete listParams.marker; listParams.keyMarker = params['key-marker']; - listParams.versionIdMarker = params['version-id-marker'] ? - versionIdUtils.decode(params['version-id-marker']) : undefined; + listParams.versionIdMarker = params['version-id-marker'] + ? versionIdUtils.decode(params['version-id-marker']) + : undefined; } if (!requestMaxKeys) { const emptyList = { @@ -128,20 +121,36 @@ function metadataSearch(authInfo, request, log, callback) { Versions: [], IsTruncated: false, }; - return handleResult(listParams, requestMaxKeys, encoding, authInfo, - bucketName, emptyList, corsHeaders, log, callback); + return handleResult( + listParams, + requestMaxKeys, + encoding, + authInfo, + bucketName, + emptyList, + corsHeaders, + log, + callback, + ); } - return services.getObjectListing(bucketName, listParams, log, - (err, list) => { - if (err) { - log.debug('error processing request', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'metadataSearch'); - return callback(err, null, corsHeaders); - } - return handleResult(listParams, requestMaxKeys, encoding, authInfo, - bucketName, list, corsHeaders, log, callback); - }); + return services.getObjectListing(bucketName, listParams, log, (err, list) => { + if (err) { + log.debug('error processing request', { error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'metadataSearch'); + return callback(err, null, corsHeaders); + } + return handleResult( + listParams, + requestMaxKeys, + encoding, + authInfo, + bucketName, + list, + corsHeaders, + log, + callback, + ); + }); }); return undefined; } diff --git a/lib/api/multipartDelete.js b/lib/api/multipartDelete.js index 48146774b7..bb43b86291 100644 --- a/lib/api/multipartDelete.js +++ b/lib/api/multipartDelete.js @@ -22,12 +22,15 @@ function multipartDelete(authInfo, request, log, callback) { const objectKey = request.objectKey; const uploadId = request.query.uploadId; - abortMultipartUpload(authInfo, bucketName, objectKey, uploadId, log, + abortMultipartUpload( + authInfo, + bucketName, + objectKey, + uploadId, + log, (err, destinationBucket, partSizeSum) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, destinationBucket); - const location = destinationBucket ? - destinationBucket.getLocationConstraint() : null; + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, destinationBucket); + const location = destinationBucket ? destinationBucket.getLocationConstraint() : null; if (err && !err?.is?.NoSuchUpload) { return callback(err, corsHeaders); } @@ -36,12 +39,10 @@ function multipartDelete(authInfo, request, log, callback) { method: 'multipartDelete', uploadId, }); - monitoring.promMetrics('DELETE', bucketName, 400, - 'abortMultipartUpload'); + monitoring.promMetrics('DELETE', bucketName, 400, 'abortMultipartUpload'); return callback(err, corsHeaders); } - monitoring.promMetrics('DELETE', bucketName, 400, - 'abortMultipartUpload'); + monitoring.promMetrics('DELETE', bucketName, 400, 'abortMultipartUpload'); if (!err) { pushMetric('abortMultipartUpload', log, { authInfo, @@ -61,7 +62,9 @@ function multipartDelete(authInfo, request, log, callback) { } } return callback(null, corsHeaders); - }, request); + }, + request, + ); } module.exports = multipartDelete; diff --git a/lib/api/objectDelete.js b/lib/api/objectDelete.js index a1901f4a5f..42375420b9 100644 --- a/lib/api/objectDelete.js +++ b/lib/api/objectDelete.js @@ -5,12 +5,10 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const services = require('../services'); const { pushMetric } = require('../utapi/utilities'); const createAndStoreObject = require('./apiUtils/object/createAndStoreObject'); -const { decodeVersionId, preprocessingVersioningDelete } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, preprocessingVersioningDelete } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const monitoring = require('../utilities/monitoringHandler'); -const { hasGovernanceBypassHeader, ObjectLockInfo } - = require('./apiUtils/object/objectLockHelpers'); +const { hasGovernanceBypassHeader, ObjectLockInfo } = require('./apiUtils/object/objectLockHelpers'); const { config } = require('../Config'); const { _deleteRequiresOplogUpdate } = require('./apiUtils/object/deleteObject'); @@ -32,8 +30,7 @@ function objectDeleteInternal(authInfo, request, log, isExpiration, cb) { log.debug('processing request', { method: 'objectDeleteInternal' }); if (authInfo.isRequesterPublicUser()) { log.debug('operation not available for public user'); - monitoring.promMetrics( - 'DELETE', request.bucketName, 403, 'deleteObject'); + monitoring.promMetrics('DELETE', request.bucketName, 403, 'deleteObject'); return cb(errors.AccessDenied); } const bucketName = request.bucketName; @@ -60,253 +57,303 @@ function objectDeleteInternal(authInfo, request, log, isExpiration, cb) { }; const canonicalID = authInfo.getCanonicalID(); - return async.waterfall([ - function validateBucketAndObj(next) { - return standardMetadataValidateBucketAndObj(valParams, request.actionImplicitDenies, log, - (err, bucketMD, objMD) => { - if (err) { - return next(err, bucketMD); + return async.waterfall( + [ + function validateBucketAndObj(next) { + return standardMetadataValidateBucketAndObj( + valParams, + request.actionImplicitDenies, + log, + (err, bucketMD, objMD) => { + if (err) { + return next(err, bucketMD); + } + + const versioningCfg = bucketMD.getVersioningConfiguration(); + if (!objMD) { + if (!versioningCfg) { + return next(errors.NoSuchKey, bucketMD); + } + // AWS does not return an error when trying to delete a + // specific version that does not exist. We skip to the end + // of the waterfall here. + if (reqVersionId) { + log.debug('trying to delete specific version ' + ' that does not exist'); + return next(errors.NoSuchVersion, bucketMD); + } + // To adhere to AWS behavior, create a delete marker even + // if trying to delete an object that does not exist when + // versioning has been configured + return next(null, bucketMD, objMD); + } + + if ( + versioningCfg && + versioningCfg.Status === 'Enabled' && + objMD.versionId === reqVersionId && + isExpiration && + !objMD.isDeleteMarker + ) { + log.warn( + 'expiration is trying to delete a master version ' + + 'of an object with versioning enabled', + { + method: 'objectDeleteInternal', + isExpiration, + reqVersionId, + versionId: objMD.versionId, + replicationState: objMD.replicationInfo, + location: objMD.location, + originOp: objMD.originOp, + }, + ); + } + if (reqVersionId && objMD.location && Array.isArray(objMD.location) && objMD.location[0]) { + // we need this information for data deletes to AWS + // eslint-disable-next-line no-param-reassign + objMD.location[0].deleteVersion = true; + } + if (objMD['content-length'] !== undefined) { + log.end().addDefaultFields({ + bytesDeleted: objMD['content-length'], + }); + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.analyticsBytesDeleted = objMD['content-length']; + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = parseInt(objMD['content-length'], 10); + } + } + return next(null, bucketMD, objMD); + }, + ); + }, + function evaluateObjectLockPolicy(bucketMD, objectMD, next) { + // AWS only returns an object lock error if a version id + // is specified, else continue to create a delete marker + if (!reqVersionId) { + return next(null, bucketMD, objectMD); } - const versioningCfg = bucketMD.getVersioningConfiguration(); - if (!objMD) { - if (!versioningCfg) { - return next(errors.NoSuchKey, bucketMD); - } - // AWS does not return an error when trying to delete a - // specific version that does not exist. We skip to the end - // of the waterfall here. - if (reqVersionId) { - log.debug('trying to delete specific version ' + - ' that does not exist'); - return next(errors.NoSuchVersion, bucketMD); - } - // To adhere to AWS behavior, create a delete marker even - // if trying to delete an object that does not exist when - // versioning has been configured - return next(null, bucketMD, objMD); + const objLockInfo = new ObjectLockInfo({ + mode: objectMD.retentionMode, + date: objectMD.retentionDate, + legalHold: objectMD.legalHold || false, + }); + + // If the object can not be deleted raise an error + if (!objLockInfo.canModifyObject(hasGovernanceBypass)) { + log.debug('trying to delete locked object'); + return next(objectLockedError, bucketMD); } - if (versioningCfg && versioningCfg.Status === 'Enabled' && - objMD.versionId === reqVersionId && isExpiration && - !objMD.isDeleteMarker) { - log.warn('expiration is trying to delete a master version ' + - 'of an object with versioning enabled', { - method: 'objectDeleteInternal', - isExpiration, - reqVersionId, - versionId: objMD.versionId, - replicationState: objMD.replicationInfo, - location: objMD.location, - originOp: objMD.originOp, - }); + return next(null, bucketMD, objectMD); + }, + function validateHeaders(bucketMD, objectMD, next) { + if (objectMD) { + const lastModified = objectMD['last-modified']; + const { modifiedSinceRes, unmodifiedSinceRes } = checkDateModifiedHeaders( + request.headers, + lastModified, + ); + const err = modifiedSinceRes.error || unmodifiedSinceRes.error; + if (err) { + return process.nextTick(() => next(err, bucketMD)); } - if (reqVersionId && objMD.location && - Array.isArray(objMD.location) && objMD.location[0]) { - // we need this information for data deletes to AWS - // eslint-disable-next-line no-param-reassign - objMD.location[0].deleteVersion = true; } - if (objMD['content-length'] !== undefined) { - log.end().addDefaultFields({ - bytesDeleted: objMD['content-length'], - }); - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.analyticsBytesDeleted = objMD['content-length']; - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = parseInt(objMD['content-length'], 10); - } + return process.nextTick(() => next(null, bucketMD, objectMD)); + }, + function deleteOperation(bucketMD, objectMD, next) { + const delOptions = preprocessingVersioningDelete( + bucketName, + bucketMD, + objectMD, + reqVersionId, + config.nullVersionCompatMode, + ); + const deleteInfo = { + removeDeleteMarker: false, + newDeleteMarker: false, + }; + if (delOptions && delOptions.deleteData && bucketMD.isNFS() && bucketMD.getReplicationConfiguration()) { + // If an NFS bucket that has replication configured, we want + // to put a delete marker on the destination even though the + // source does not have versioning. + return createAndStoreObject( + bucketName, + bucketMD, + objectKey, + objectMD, + authInfo, + canonicalID, + null, + request, + true, + null, + log, + isExpiration + ? 's3:LifecycleExpiration:DeleteMarkerCreated' + : 's3:ObjectRemoved:DeleteMarkerCreated', + err => { + if (err) { + return next(err); + } + if (objectMD.isDeleteMarker) { + // record that we deleted a delete marker to set + // response headers accordingly + deleteInfo.removeDeleteMarker = true; + } + return services.deleteObject( + bucketName, + objectMD, + objectKey, + delOptions, + false, + log, + isExpiration ? 's3:LifecycleExpiration:Delete' : 's3:ObjectRemoved:Delete', + (err, delResult) => next(err, bucketMD, objectMD, delResult, deleteInfo), + ); + }, + ); } - return next(null, bucketMD, objMD); - }); - }, - function evaluateObjectLockPolicy(bucketMD, objectMD, next) { - // AWS only returns an object lock error if a version id - // is specified, else continue to create a delete marker - if (!reqVersionId) { - return next(null, bucketMD, objectMD); - } - - const objLockInfo = new ObjectLockInfo({ - mode: objectMD.retentionMode, - date: objectMD.retentionDate, - legalHold: objectMD.legalHold || false, - }); + if (delOptions && delOptions.deleteData) { + delOptions.overheadField = overheadField; + if (objectMD.isDeleteMarker) { + // record that we deleted a delete marker to set + // response headers accordingly + deleteInfo.removeDeleteMarker = true; + } - // If the object can not be deleted raise an error - if (!objLockInfo.canModifyObject(hasGovernanceBypass)) { - log.debug('trying to delete locked object'); - return next(objectLockedError, bucketMD); - } + if (objectMD.uploadId) { + delOptions.replayId = objectMD.uploadId; + } - return next(null, bucketMD, objectMD); - }, - function validateHeaders(bucketMD, objectMD, next) { - if (objectMD) { - const lastModified = objectMD['last-modified']; - const { modifiedSinceRes, unmodifiedSinceRes } = - checkDateModifiedHeaders(request.headers, lastModified); - const err = modifiedSinceRes.error || unmodifiedSinceRes.error; - if (err) { - return process.nextTick(() => next(err, bucketMD)); - } - } - return process.nextTick(() => - next(null, bucketMD, objectMD)); - }, - function deleteOperation(bucketMD, objectMD, next) { - const delOptions = preprocessingVersioningDelete( - bucketName, bucketMD, objectMD, reqVersionId, config.nullVersionCompatMode); - const deleteInfo = { - removeDeleteMarker: false, - newDeleteMarker: false, - }; - if (delOptions && delOptions.deleteData && bucketMD.isNFS() && - bucketMD.getReplicationConfiguration()) { - // If an NFS bucket that has replication configured, we want - // to put a delete marker on the destination even though the - // source does not have versioning. - return createAndStoreObject(bucketName, bucketMD, objectKey, - objectMD, authInfo, canonicalID, null, request, true, null, - log, isExpiration ? - 's3:LifecycleExpiration:DeleteMarkerCreated' : - 's3:ObjectRemoved:DeleteMarkerCreated', - err => { - if (err) { - return next(err); - } - if (objectMD.isDeleteMarker) { - // record that we deleted a delete marker to set - // response headers accordingly - deleteInfo.removeDeleteMarker = true; - } - return services.deleteObject(bucketName, objectMD, - objectKey, delOptions, false, log, isExpiration ? - 's3:LifecycleExpiration:Delete' : - 's3:ObjectRemoved:Delete', - (err, delResult) => - next(err, bucketMD, objectMD, delResult, deleteInfo)); - }); - } - if (delOptions && delOptions.deleteData) { - delOptions.overheadField = overheadField; - if (objectMD.isDeleteMarker) { - // record that we deleted a delete marker to set - // response headers accordingly - deleteInfo.removeDeleteMarker = true; - } + if (!_deleteRequiresOplogUpdate(objectMD, bucketMD)) { + delOptions.doesNotNeedOpogUpdate = true; + } - if (objectMD.uploadId) { - delOptions.replayId = objectMD.uploadId; + return services.deleteObject( + bucketName, + objectMD, + objectKey, + delOptions, + false, + log, + isExpiration ? 's3:LifecycleExpiration:Delete' : 's3:ObjectRemoved:Delete', + (err, delResult) => next(err, bucketMD, objectMD, delResult, deleteInfo), + ); } - - if (!_deleteRequiresOplogUpdate(objectMD, bucketMD)) { - delOptions.doesNotNeedOpogUpdate = true; + // putting a new delete marker + deleteInfo.newDeleteMarker = true; + return createAndStoreObject( + bucketName, + bucketMD, + objectKey, + objectMD, + authInfo, + canonicalID, + null, + request, + deleteInfo.newDeleteMarker, + null, + overheadField, + log, + isExpiration + ? 's3:LifecycleExpiration:DeleteMarkerCreated' + : 's3:ObjectRemoved:DeleteMarkerCreated', + (err, newDelMarkerRes) => { + next(err, bucketMD, objectMD, newDelMarkerRes, deleteInfo); + }, + ); + }, + ], + (err, bucketMD, objectMD, result, deleteInfo) => { + const resHeaders = collectCorsHeaders(request.headers.origin, request.method, bucketMD); + // if deleting a specific version or delete marker, return version id + // in the response headers, even in case of NoSuchVersion + if (reqVersionId) { + resHeaders['x-amz-version-id'] = + reqVersionId === 'null' ? reqVersionId : versionIdUtils.encode(reqVersionId); + if (deleteInfo && deleteInfo.removeDeleteMarker) { + resHeaders['x-amz-delete-marker'] = true; } - - return services.deleteObject(bucketName, objectMD, objectKey, - delOptions, false, log, isExpiration ? - 's3:LifecycleExpiration:Delete' : - 's3:ObjectRemoved:Delete', - (err, delResult) => next(err, bucketMD, - objectMD, delResult, deleteInfo)); - } - // putting a new delete marker - deleteInfo.newDeleteMarker = true; - return createAndStoreObject(bucketName, bucketMD, - objectKey, objectMD, authInfo, canonicalID, null, request, - deleteInfo.newDeleteMarker, null, overheadField, log, isExpiration ? - 's3:LifecycleExpiration:DeleteMarkerCreated' : - 's3:ObjectRemoved:DeleteMarkerCreated', - (err, newDelMarkerRes) => { - next(err, bucketMD, objectMD, newDelMarkerRes, deleteInfo); - }); - }, - ], (err, bucketMD, objectMD, result, deleteInfo) => { - const resHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucketMD); - // if deleting a specific version or delete marker, return version id - // in the response headers, even in case of NoSuchVersion - if (reqVersionId) { - resHeaders['x-amz-version-id'] = reqVersionId === 'null' ? - reqVersionId : versionIdUtils.encode(reqVersionId); - if (deleteInfo && deleteInfo.removeDeleteMarker) { - resHeaders['x-amz-delete-marker'] = true; } - } - if (err === objectLockedError) { - log.debug('preventing deletion due to object lock', - { + if (err === objectLockedError) { + log.debug('preventing deletion due to object lock', { error: errors.AccessDenied, objectLocked: true, method: 'objectDelete', }); - return cb(errors.AccessDenied, resHeaders); - } - if (err) { - log.debug('error processing request', { error: err, - method: 'objectDelete' }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteObject'); - return cb(err, resHeaders); - } - if (deleteInfo.newDeleteMarker) { - // if we created a new delete marker, return true for - // x-amz-delete-marker and the version ID of the new delete marker - if (result.versionId) { - resHeaders['x-amz-delete-marker'] = true; - resHeaders['x-amz-version-id'] = result.versionId === 'null' ? - result.versionId : versionIdUtils.encode(result.versionId); + return cb(errors.AccessDenied, resHeaders); } + if (err) { + log.debug('error processing request', { error: err, method: 'objectDelete' }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteObject'); + return cb(err, resHeaders); + } + if (deleteInfo.newDeleteMarker) { + // if we created a new delete marker, return true for + // x-amz-delete-marker and the version ID of the new delete marker + if (result.versionId) { + resHeaders['x-amz-delete-marker'] = true; + resHeaders['x-amz-version-id'] = + result.versionId === 'null' ? result.versionId : versionIdUtils.encode(result.versionId); + } - /* byteLength is passed under the following conditions: - * - bucket versioning is suspended - * - object version id is null - * and one of: - * - the content length of the object exists - * - or - - * - it is a delete marker - * In this case, the master key is deleted and replaced with a delete marker. - * The decrement accounts for the deletion of the master key when utapi reports - * on the number of objects. - */ - // FIXME: byteLength may be incorrect, see S3C-7440 - const versioningSuspended = bucketMD.getVersioningConfiguration() - && bucketMD.getVersioningConfiguration().Status === 'Suspended'; - const deletedSuspendedMasterVersion = versioningSuspended && !!objectMD; - // Default to 0 content-length to cover deleting a DeleteMarker - const objectByteLength = (objectMD && objectMD['content-length']) || 0; - const byteLength = deletedSuspendedMasterVersion ? Number.parseInt(objectByteLength, 10) : null; + /* byteLength is passed under the following conditions: + * - bucket versioning is suspended + * - object version id is null + * and one of: + * - the content length of the object exists + * - or - + * - it is a delete marker + * In this case, the master key is deleted and replaced with a delete marker. + * The decrement accounts for the deletion of the master key when utapi reports + * on the number of objects. + */ + // FIXME: byteLength may be incorrect, see S3C-7440 + const versioningSuspended = + bucketMD.getVersioningConfiguration() && + bucketMD.getVersioningConfiguration().Status === 'Suspended'; + const deletedSuspendedMasterVersion = versioningSuspended && !!objectMD; + // Default to 0 content-length to cover deleting a DeleteMarker + const objectByteLength = (objectMD && objectMD['content-length']) || 0; + const byteLength = deletedSuspendedMasterVersion ? Number.parseInt(objectByteLength, 10) : null; - pushMetric('putDeleteMarkerObject', log, { - authInfo, - byteLength, - bucket: bucketName, - keys: [objectKey], - versionId: result.versionId, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - } else { - log.end().addDefaultFields({ - contentLength: objectMD['content-length'], - }); - pushMetric('deleteObject', log, { - authInfo, - canonicalID: bucketMD.getOwner(), - bucket: bucketName, - keys: [objectKey], - byteLength: Number.parseInt(objectMD['content-length'], 10), - numberOfObjects: 1, - location: objectMD.dataStoreName, - isDelete: true, - }); - monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObject', - Number.parseInt(objectMD['content-length'], 10)); - } - return cb(err, resHeaders); - }); + pushMetric('putDeleteMarkerObject', log, { + authInfo, + byteLength, + bucket: bucketName, + keys: [objectKey], + versionId: result.versionId, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + } else { + log.end().addDefaultFields({ + contentLength: objectMD['content-length'], + }); + pushMetric('deleteObject', log, { + authInfo, + canonicalID: bucketMD.getOwner(), + bucket: bucketName, + keys: [objectKey], + byteLength: Number.parseInt(objectMD['content-length'], 10), + numberOfObjects: 1, + location: objectMD.dataStoreName, + isDelete: true, + }); + monitoring.promMetrics( + 'DELETE', + bucketName, + '200', + 'deleteObject', + Number.parseInt(objectMD['content-length'], 10), + ); + } + return cb(err, resHeaders); + }, + ); } /** diff --git a/lib/api/objectGet.js b/lib/api/objectGet.js index 319fe67db4..cd865d8845 100644 --- a/lib/api/objectGet.js +++ b/lib/api/objectGet.js @@ -10,10 +10,8 @@ const collectResponseHeaders = require('../utilities/collectResponseHeaders'); const { pushMetric } = require('../utapi/utilities'); const { getVersionIdResHeader } = require('./apiUtils/object/versioning'); const setPartRanges = require('./apiUtils/object/setPartRanges'); -const locationHeaderCheck = - require('./apiUtils/object/locationHeaderCheck'); -const getReplicationBackendDataLocator = - require('./apiUtils/object/getReplicationBackendDataLocator'); +const locationHeaderCheck = require('./apiUtils/object/locationHeaderCheck'); +const getReplicationBackendDataLocator = require('./apiUtils/object/getReplicationBackendDataLocator'); const checkReadLocation = require('./apiUtils/object/checkReadLocation'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -48,8 +46,7 @@ function objectGet(authInfo, request, returnTagCount, log, callback) { } // returns name of location to get from and key if successful - const locCheckResult = - locationHeaderCheck(request.headers, objectKey, bucketName); + const locCheckResult = locationHeaderCheck(request.headers, objectKey, bucketName); if (locCheckResult instanceof Error) { log.trace('invalid location constraint to get from', { location: request.headers['x-amz-location-constraint'], @@ -79,303 +76,277 @@ function objectGet(authInfo, request, returnTagCount, log, callback) { returnTagCount, }; - return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, - (err, bucket, objMD) => updateEncryption(err, bucket, objMD, objectKey, log, {}, - (err, bucket, objMD) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.debug('error processing request', { - error: err, - method: 'metadataValidateBucketAndObj', - }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, null, corsHeaders); - } - if (!objMD) { - const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, null, corsHeaders); - } - const verCfg = bucket.getVersioningConfiguration(); - // check if object data is in a cold storage - const coldErr = verifyColdObjectAvailable(objMD); - if (coldErr) { - monitoring.promMetrics( - 'GET', bucketName, coldErr.code, 'getObject'); - return callback(coldErr, null, corsHeaders); - } - if (objMD.isDeleteMarker) { - const responseMetaHeaders = Object.assign({}, - { 'x-amz-delete-marker': true }, corsHeaders); - if (!versionId) { - monitoring.promMetrics( - 'GET', bucketName, 404, 'getObject'); - return callback(errors.NoSuchKey, null, responseMetaHeaders); + return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, (err, bucket, objMD) => + updateEncryption(err, bucket, objMD, objectKey, log, {}, (err, bucket, objMD) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.debug('error processing request', { + error: err, + method: 'metadataValidateBucketAndObj', + }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, null, corsHeaders); } - // return MethodNotAllowed if requesting a specific - // version that has a delete marker - responseMetaHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objMD); - monitoring.promMetrics( - 'GET', bucketName, 405, 'getObject'); - return callback(errors.MethodNotAllowed, null, - responseMetaHeaders); - } - const headerValResult = validateHeaders(request.headers, - objMD['last-modified'], objMD['content-md5']); - if (headerValResult.error) { - return callback(headerValResult.error, null, corsHeaders); - } - const responseMetaHeaders = collectResponseHeaders(objMD, - corsHeaders, verCfg, - returnTagCount && objMD.returnTagCount); // IAM and Bucket policy should both authorize tagging. - - setExpirationHeaders(responseMetaHeaders, { - lifecycleConfig: bucket.getLifecycleConfiguration(), - objectParams: { - key: objectKey, - tags: objMD.tags, - date: objMD['last-modified'], - }, - isVersionedReq: !!versionId, - }); - - const objLength = (objMD.location === null ? - 0 : parseInt(objMD['content-length'], 10)); - // Store full object size for server access logs - if (request.serverAccessLog) { - // eslint-disable-next-line no-param-reassign - request.serverAccessLog.objectSize = objLength; - } - let byteRange; - let partNumber = null; - const streamingParams = {}; - if (request.headers.range) { - const { range, error } = parseRange(request.headers.range, - objLength); - if (error) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(error, null, corsHeaders); + if (!objMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, null, corsHeaders); } - responseMetaHeaders['Accept-Ranges'] = 'bytes'; - if (range) { - byteRange = range; - // End of range should be included so + 1 - responseMetaHeaders['Content-Length'] = - range[1] - range[0] + 1; - responseMetaHeaders['Content-Range'] = - `bytes ${range[0]}-${range[1]}/${objLength}`; - streamingParams.rangeStart = (range[0] || typeof range[0] === 'number') ? - range[0].toString() : undefined; - streamingParams.rangeEnd = range[1] ? - range[1].toString() : undefined; + const verCfg = bucket.getVersioningConfiguration(); + // check if object data is in a cold storage + const coldErr = verifyColdObjectAvailable(objMD); + if (coldErr) { + monitoring.promMetrics('GET', bucketName, coldErr.code, 'getObject'); + return callback(coldErr, null, corsHeaders); } - } - let dataLocator = null; - if (objMD.location !== null) { - // To provide for backwards compatibility before - // md-model-version 2, need to handle cases where - // objMD.location is just a string - dataLocator = Array.isArray(objMD.location) ? - objMD.location : [{ key: objMD.location }]; - - const repConf = bucket.getReplicationConfiguration(); - const prefReadLocation = repConf && repConf.preferredReadLocation; - const prefReadDataLocator = checkReadLocation(config, - prefReadLocation, objectKey, bucketName); - const targetLocation = locCheckResult || prefReadDataLocator || - null; - - if (targetLocation && - targetLocation.location !== objMD.dataStoreName) { - const repBackendResult = getReplicationBackendDataLocator( - targetLocation, objMD.replicationInfo); - if (repBackendResult.error) { - log.error('Error with location constraint header', { - bucketName, objectKey, versionId, - error: repBackendResult.error, - status: repBackendResult.status, - }); - return callback(repBackendResult.error, null, corsHeaders); - } - const targetDataLocator = repBackendResult.dataLocator; - if (targetDataLocator) { - dataLocator = targetDataLocator; - } else { - log.debug('using source location as preferred read ' + - 'is unavailable', { - bucketName, objectKey, versionId, - reason: repBackendResult.reason, - }); + if (objMD.isDeleteMarker) { + const responseMetaHeaders = Object.assign({}, { 'x-amz-delete-marker': true }, corsHeaders); + if (!versionId) { + monitoring.promMetrics('GET', bucketName, 404, 'getObject'); + return callback(errors.NoSuchKey, null, responseMetaHeaders); } + // return MethodNotAllowed if requesting a specific + // version that has a delete marker + responseMetaHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objMD); + monitoring.promMetrics('GET', bucketName, 405, 'getObject'); + return callback(errors.MethodNotAllowed, null, responseMetaHeaders); } - // if the data backend is azure, there will only ever be at - // most one item in the dataLocator array - if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') { - dataLocator[0].azureStreamingOptions = streamingParams; + const headerValResult = validateHeaders(request.headers, objMD['last-modified'], objMD['content-md5']); + if (headerValResult.error) { + return callback(headerValResult.error, null, corsHeaders); } - if (request.query && request.query.partNumber !== undefined) { - if (byteRange) { - const error = errorInstances.InvalidRequest - .customizeDescription('Cannot specify both Range ' + - 'header and partNumber query parameter.'); - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(error, null, corsHeaders); - } - partNumber = Number.parseInt(request.query.partNumber, 10); - if (Number.isNaN(partNumber)) { - const error = errorInstances.InvalidArgument - .customizeDescription('Part number must be a number.'); - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(error, null, corsHeaders); - } - if (partNumber < 1 || partNumber > 10000) { - const error = errorInstances.InvalidArgument - .customizeDescription('Part number must be an ' + - 'integer between 1 and 10000, inclusive.'); - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); + const responseMetaHeaders = collectResponseHeaders( + objMD, + corsHeaders, + verCfg, + returnTagCount && objMD.returnTagCount, + ); // IAM and Bucket policy should both authorize tagging. + + setExpirationHeaders(responseMetaHeaders, { + lifecycleConfig: bucket.getLifecycleConfiguration(), + objectParams: { + key: objectKey, + tags: objMD.tags, + date: objMD['last-modified'], + }, + isVersionedReq: !!versionId, + }); + + const objLength = objMD.location === null ? 0 : parseInt(objMD['content-length'], 10); + // Store full object size for server access logs + if (request.serverAccessLog) { + // eslint-disable-next-line no-param-reassign + request.serverAccessLog.objectSize = objLength; + } + let byteRange; + let partNumber = null; + const streamingParams = {}; + if (request.headers.range) { + const { range, error } = parseRange(request.headers.range, objLength); + if (error) { + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); return callback(error, null, corsHeaders); } - } - // If have a data model before version 2, cannot support - // get range for objects with multiple parts - if (byteRange && dataLocator.length > 1 && - dataLocator[0].start === undefined) { - monitoring.promMetrics( - 'GET', bucketName, 501, 'getObject'); - return callback(errors.NotImplemented, null, corsHeaders); - } - if (objMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - objMD['x-amz-server-side-encryption-aws-kms-key-id']; - dataLocator[i].algorithm = - objMD['x-amz-server-side-encryption']; + responseMetaHeaders['Accept-Ranges'] = 'bytes'; + if (range) { + byteRange = range; + // End of range should be included so + 1 + responseMetaHeaders['Content-Length'] = range[1] - range[0] + 1; + responseMetaHeaders['Content-Range'] = `bytes ${range[0]}-${range[1]}/${objLength}`; + streamingParams.rangeStart = + range[0] || typeof range[0] === 'number' ? range[0].toString() : undefined; + streamingParams.rangeEnd = range[1] ? range[1].toString() : undefined; } } - if (partNumber) { - const locations = []; - let locationPartNumber; - for (let i = 0; i < objMD.location.length; i++) { - const { dataStoreETag } = objMD.location[i]; + let dataLocator = null; + if (objMD.location !== null) { + // To provide for backwards compatibility before + // md-model-version 2, need to handle cases where + // objMD.location is just a string + dataLocator = Array.isArray(objMD.location) ? objMD.location : [{ key: objMD.location }]; - if (dataStoreETag) { - locationPartNumber = - Number.parseInt(dataStoreETag.split(':')[0], 10); + const repConf = bucket.getReplicationConfiguration(); + const prefReadLocation = repConf && repConf.preferredReadLocation; + const prefReadDataLocator = checkReadLocation(config, prefReadLocation, objectKey, bucketName); + const targetLocation = locCheckResult || prefReadDataLocator || null; + + if (targetLocation && targetLocation.location !== objMD.dataStoreName) { + const repBackendResult = getReplicationBackendDataLocator(targetLocation, objMD.replicationInfo); + if (repBackendResult.error) { + log.error('Error with location constraint header', { + bucketName, + objectKey, + versionId, + error: repBackendResult.error, + status: repBackendResult.status, + }); + return callback(repBackendResult.error, null, corsHeaders); + } + const targetDataLocator = repBackendResult.dataLocator; + if (targetDataLocator) { + dataLocator = targetDataLocator; } else { - /** - * Location objects prior to GA7.1 do not include the - * dataStoreETag field so we cannot find the part range, - * the objects are treated as if they only have 1 part - */ - locationPartNumber = 1; + log.debug('using source location as preferred read ' + 'is unavailable', { + bucketName, + objectKey, + versionId, + reason: repBackendResult.reason, + }); } - - // Get all parts that belong to the requested part number - if (partNumber === locationPartNumber) { - locations.push(objMD.location[i]); - } else if (locationPartNumber > partNumber) { - break; + } + // if the data backend is azure, there will only ever be at + // most one item in the dataLocator array + if (dataLocator[0] && dataLocator[0].dataStoreType === 'azure') { + dataLocator[0].azureStreamingOptions = streamingParams; + } + if (request.query && request.query.partNumber !== undefined) { + if (byteRange) { + const error = errorInstances.InvalidRequest.customizeDescription( + 'Cannot specify both Range ' + 'header and partNumber query parameter.', + ); + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(error, null, corsHeaders); + } + partNumber = Number.parseInt(request.query.partNumber, 10); + if (Number.isNaN(partNumber)) { + const error = errorInstances.InvalidArgument.customizeDescription( + 'Part number must be a number.', + ); + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(error, null, corsHeaders); + } + if (partNumber < 1 || partNumber > 10000) { + const error = errorInstances.InvalidArgument.customizeDescription( + 'Part number must be an ' + 'integer between 1 and 10000, inclusive.', + ); + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(error, null, corsHeaders); } } - if (locations.length === 0) { - monitoring.promMetrics( - 'GET', bucketName, 400, 'getObject'); - return callback(errors.InvalidPartNumber, null, - corsHeaders); + // If have a data model before version 2, cannot support + // get range for objects with multiple parts + if (byteRange && dataLocator.length > 1 && dataLocator[0].start === undefined) { + monitoring.promMetrics('GET', bucketName, 501, 'getObject'); + return callback(errors.NotImplemented, null, corsHeaders); } - const { start } = locations[0]; - const endLocation = locations[locations.length - 1]; - const end = endLocation.start + endLocation.size - 1; - responseMetaHeaders['Content-Length'] = end - start + 1; - const partByteRange = [start, end]; - dataLocator = setPartRanges(dataLocator, partByteRange); - const partsCount = getPartCountFromMd5(objMD); - if (partsCount) { - responseMetaHeaders['x-amz-mp-parts-count'] = - partsCount; + if (objMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = objMD['x-amz-server-side-encryption-aws-kms-key-id']; + dataLocator[i].algorithm = objMD['x-amz-server-side-encryption']; + } + } + if (partNumber) { + const locations = []; + let locationPartNumber; + for (let i = 0; i < objMD.location.length; i++) { + const { dataStoreETag } = objMD.location[i]; + + if (dataStoreETag) { + locationPartNumber = Number.parseInt(dataStoreETag.split(':')[0], 10); + } else { + /** + * Location objects prior to GA7.1 do not include the + * dataStoreETag field so we cannot find the part range, + * the objects are treated as if they only have 1 part + */ + locationPartNumber = 1; + } + + // Get all parts that belong to the requested part number + if (partNumber === locationPartNumber) { + locations.push(objMD.location[i]); + } else if (locationPartNumber > partNumber) { + break; + } + } + if (locations.length === 0) { + monitoring.promMetrics('GET', bucketName, 400, 'getObject'); + return callback(errors.InvalidPartNumber, null, corsHeaders); + } + const { start } = locations[0]; + const endLocation = locations[locations.length - 1]; + const end = endLocation.start + endLocation.size - 1; + responseMetaHeaders['Content-Length'] = end - start + 1; + const partByteRange = [start, end]; + dataLocator = setPartRanges(dataLocator, partByteRange); + const partsCount = getPartCountFromMd5(objMD); + if (partsCount) { + responseMetaHeaders['x-amz-mp-parts-count'] = partsCount; + } + } else { + dataLocator = setPartRanges(dataLocator, byteRange); } - } else { - dataLocator = setPartRanges(dataLocator, byteRange); - } - } - // Checksum is not returned for partNumber requests because per-part - // checksums are not yet stored (S3C-11073). - if (checksumMode === 'ENABLED' && !byteRange && !partNumber) { - const checksum = objMD.checksum; - if (checksum) { - responseMetaHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`] - = checksum.checksumValue; - responseMetaHeaders['x-amz-checksum-type'] = checksum.checksumType; } - } - // Check KMS Key access and usability before checking data - // diff with AWS: for empty object (no dataLocator) KMS not checked - return async.each(dataLocator || [], - (objectGetInfo, next) => { - if (!objectGetInfo.cipheredDataKey) { - return next(); + // Checksum is not returned for partNumber requests because per-part + // checksums are not yet stored (S3C-11073). + if (checksumMode === 'ENABLED' && !byteRange && !partNumber) { + const checksum = objMD.checksum; + if (checksum) { + responseMetaHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`] = checksum.checksumValue; + responseMetaHeaders['x-amz-checksum-type'] = checksum.checksumType; } - const serverSideEncryption = { - cryptoScheme: objectGetInfo.cryptoScheme, - masterKeyId: objectGetInfo.masterKeyId, - cipheredDataKey: Buffer.from( - objectGetInfo.cipheredDataKey, 'base64'), - }; - const offset = objectGetInfo.range ? objectGetInfo.range[0] : 0; - return kms.createDecipherBundle(serverSideEncryption, - offset, log, (err, decipherBundle) => { + } + // Check KMS Key access and usability before checking data + // diff with AWS: for empty object (no dataLocator) KMS not checked + return async.each( + dataLocator || [], + (objectGetInfo, next) => { + if (!objectGetInfo.cipheredDataKey) { + return next(); + } + const serverSideEncryption = { + cryptoScheme: objectGetInfo.cryptoScheme, + masterKeyId: objectGetInfo.masterKeyId, + cipheredDataKey: Buffer.from(objectGetInfo.cipheredDataKey, 'base64'), + }; + const offset = objectGetInfo.range ? objectGetInfo.range[0] : 0; + return kms.createDecipherBundle(serverSideEncryption, offset, log, (err, decipherBundle) => { if (err) { - log.error('cannot get decipher bundle from kms', - { method: 'objectGet' }); + log.error('cannot get decipher bundle from kms', { method: 'objectGet' }); return next(err); } // eslint-disable-next-line no-param-reassign objectGetInfo.decipherStream = decipherBundle.decipher; return next(); }); - }, - err => { - if (err) { - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err); - } - - return data.head(dataLocator, log, err => { + }, + err => { if (err) { - if (!err.is.LocationNotFound) { - log.error('error from external backend checking for ' + - 'object existence', { error: err }); - } - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); return callback(err); } - pushMetric('getObject', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - newByteLength: + + return data.head(dataLocator, log, err => { + if (err) { + if (!err.is.LocationNotFound) { + log.error('error from external backend checking for ' + 'object existence', { + error: err, + }); + } + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err); + } + pushMetric('getObject', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + newByteLength: Number.parseInt(responseMetaHeaders['Content-Length'], 10), + versionId: objMD.versionId, + location: objMD.dataStoreName, + }); + monitoring.promMetrics( + 'GET', + bucketName, + '200', + 'getObject', Number.parseInt(responseMetaHeaders['Content-Length'], 10), - versionId: objMD.versionId, - location: objMD.dataStoreName, + ); + return callback(null, dataLocator, responseMetaHeaders, byteRange); }); - monitoring.promMetrics('GET', bucketName, '200', 'getObject', - Number.parseInt(responseMetaHeaders['Content-Length'], 10)); - return callback(null, dataLocator, responseMetaHeaders, - byteRange); - }); - } - ); - })); + }, + ); + }), + ); } module.exports = objectGet; diff --git a/lib/api/objectGetACL.js b/lib/api/objectGetACL.js index 5f4045bbc0..030ebb8e22 100644 --- a/lib/api/objectGetACL.js +++ b/lib/api/objectGetACL.js @@ -4,8 +4,7 @@ const { errors } = require('arsenal'); const aclUtils = require('../utilities/aclUtils'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const { pushMetric } = require('../utapi/utilities'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const vault = require('../auth/vault'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const monitoring = require('../utilities/monitoringHandler'); @@ -72,128 +71,123 @@ function objectGetACL(authInfo, request, log, callback) { }, }; - return async.waterfall([ - function validateBucketAndObj(next) { - return standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectGetACL', error: err }); - return next(err); - } - if (!objectMD) { - const err = versionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error processing request', - { method: 'objectGetACL', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - if (versionId) { - log.trace('requested version is delete marker', - { method: 'objectGetACL' }); + return async.waterfall( + [ + function validateBucketAndObj(next) { + return standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectGetACL', error: err }); + return next(err); + } + if (!objectMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error processing request', { method: 'objectGetACL', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + if (versionId) { + log.trace('requested version is delete marker', { method: 'objectGetACL' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed); + } + log.trace('most recent version is delete marker', { method: 'objectGetACL' }); // FIXME we should return a `x-amz-delete-marker: true` header, // see S3C-7592 - return next(errors.MethodNotAllowed); + return next(errors.NoSuchKey); } - log.trace('most recent version is delete marker', - { method: 'objectGetACL' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.NoSuchKey); + return next(null, bucket, objectMD); + }, + ); + }, + function gatherACLs(bucket, objectMD, next) { + const verCfg = bucket.getVersioningConfiguration(); + const resVersionId = getVersionIdResHeader(verCfg, objectMD); + const objectACL = objectMD.acl; + grantInfo.ownerInfo.ID = objectMD['owner-id']; + grantInfo.ownerInfo.displayName = objectMD['owner-display-name']; + // Object owner always has full control + const ownerGrant = { + ID: objectMD['owner-id'], + displayName: objectMD['owner-display-name'], + permission: 'FULL_CONTROL', + }; + if (objectACL.Canned !== '') { + /** + * If bucket owner and object owner are different + * need to send info about bucket owner from bucket + * metadata to handleCannedGrant function + */ + let cannedGrants; + if (bucket.getOwner() !== objectMD['owner-id']) { + cannedGrants = aclUtils.handleCannedGrant(objectACL.Canned, ownerGrant, bucket); + } else { + cannedGrants = aclUtils.handleCannedGrant(objectACL.Canned, ownerGrant); } - return next(null, bucket, objectMD); - }); - }, - function gatherACLs(bucket, objectMD, next) { - const verCfg = bucket.getVersioningConfiguration(); - const resVersionId = getVersionIdResHeader(verCfg, objectMD); - const objectACL = objectMD.acl; - grantInfo.ownerInfo.ID = objectMD['owner-id']; - grantInfo.ownerInfo.displayName = objectMD['owner-display-name']; - // Object owner always has full control - const ownerGrant = { - ID: objectMD['owner-id'], - displayName: objectMD['owner-display-name'], - permission: 'FULL_CONTROL', - }; - if (objectACL.Canned !== '') { - /** - * If bucket owner and object owner are different - * need to send info about bucket owner from bucket - * metadata to handleCannedGrant function - */ - let cannedGrants; - if (bucket.getOwner() !== objectMD['owner-id']) { - cannedGrants = aclUtils.handleCannedGrant( - objectACL.Canned, ownerGrant, bucket); - } else { - cannedGrants = aclUtils.handleCannedGrant( - objectACL.Canned, ownerGrant); + grantInfo.grants = grantInfo.grants.concat(cannedGrants); + const xml = aclUtils.convertToXml(grantInfo); + return next(null, bucket, xml, resVersionId); } - grantInfo.grants = grantInfo.grants.concat(cannedGrants); - const xml = aclUtils.convertToXml(grantInfo); - return next(null, bucket, xml, resVersionId); - } - /** - * Build array of all canonicalIDs used in ACLs so duplicates - * will be retained (e.g. if an account has both read and write - * privileges, want to display both and not lose the duplicate - * when receive one dictionary entry back from Vault) - */ - const canonicalIDs = aclUtils.getCanonicalIDs(objectACL); - // Build array with grants by URI - const uriGrantInfo = aclUtils.getUriGrantInfo(objectACL); + /** + * Build array of all canonicalIDs used in ACLs so duplicates + * will be retained (e.g. if an account has both read and write + * privileges, want to display both and not lose the duplicate + * when receive one dictionary entry back from Vault) + */ + const canonicalIDs = aclUtils.getCanonicalIDs(objectACL); + // Build array with grants by URI + const uriGrantInfo = aclUtils.getUriGrantInfo(objectACL); - if (canonicalIDs.length === 0) { + if (canonicalIDs.length === 0) { + /** + * If no acl's set by account canonicalID, just add URI + * grants (if any) and return + */ + grantInfo.grants = grantInfo.grants.concat(uriGrantInfo); + const xml = aclUtils.convertToXml(grantInfo); + return next(null, bucket, xml, resVersionId); + } /** - * If no acl's set by account canonicalID, just add URI - * grants (if any) and return - */ - grantInfo.grants = grantInfo.grants.concat(uriGrantInfo); - const xml = aclUtils.convertToXml(grantInfo); - return next(null, bucket, xml, resVersionId); + * If acl's set by account canonicalID, + * get emails from Vault to serve + * as display names + */ + return vault.getEmailAddresses(canonicalIDs, log, (err, emails) => { + if (err) { + log.trace('error processing request', { method: 'objectGetACL', error: err }); + return next(err, bucket); + } + const individualGrants = aclUtils.getIndividualGrants(objectACL, canonicalIDs, emails); + // Add to grantInfo any individual grants and grants by uri + grantInfo.grants = grantInfo.grants.concat(individualGrants).concat(uriGrantInfo); + // parse info about accounts and owner info to convert to xml + const xml = aclUtils.convertToXml(grantInfo); + return next(null, bucket, xml, resVersionId); + }); + }, + ], + (err, bucket, xml, resVersionId) => { + const resHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + monitoring.promMetrics('GET', bucketName, err.code, 'getObjectAcl'); + return callback(err, null, resHeaders); } - /** - * If acl's set by account canonicalID, - * get emails from Vault to serve - * as display names - */ - return vault.getEmailAddresses(canonicalIDs, log, (err, emails) => { - if (err) { - log.trace('error processing request', - { method: 'objectGetACL', error: err }); - return next(err, bucket); - } - const individualGrants = aclUtils.getIndividualGrants(objectACL, - canonicalIDs, emails); - // Add to grantInfo any individual grants and grants by uri - grantInfo.grants = grantInfo.grants - .concat(individualGrants).concat(uriGrantInfo); - // parse info about accounts and owner info to convert to xml - const xml = aclUtils.convertToXml(grantInfo); - return next(null, bucket, xml, resVersionId); + pushMetric('getObjectAcl', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: resVersionId, + location: bucket ? bucket.getLocationConstraint() : undefined, }); + monitoring.promMetrics('GET', bucketName, '200', 'getObjectAcl'); + resHeaders['x-amz-version-id'] = resVersionId; + return callback(null, xml, resHeaders); }, - ], (err, bucket, xml, resVersionId) => { - const resHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObjectAcl'); - return callback(err, null, resHeaders); - } - pushMetric('getObjectAcl', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: resVersionId, - location: bucket ? bucket.getLocationConstraint() : undefined, - }); - monitoring.promMetrics('GET', bucketName, '200', 'getObjectAcl'); - resHeaders['x-amz-version-id'] = resVersionId; - return callback(null, xml, resHeaders); - }); + ); } module.exports = objectGetACL; diff --git a/lib/api/objectGetAttributes.js b/lib/api/objectGetAttributes.js index 9ef71eabbb..61b8e4e78b 100644 --- a/lib/api/objectGetAttributes.js +++ b/lib/api/objectGetAttributes.js @@ -21,10 +21,7 @@ const OBJECT_GET_ATTRIBUTES = 'objectGetAttributes'; * @returns {string} XML response */ function buildXmlResponse(objMD, requestedAttrs, log) { - const xml = [ - '', - '', - ]; + const xml = ['', '']; const userMetadata = getUserMetadata(objMD); buildAttributesXml(objMD, userMetadata, requestedAttrs, xml, log); @@ -33,7 +30,6 @@ function buildXmlResponse(objMD, requestedAttrs, log) { return xml.join(''); } - /** * getUserMetadata - Retrieves all object user metadata * @param {object} objMD - object metadata diff --git a/lib/api/objectGetLegalHold.js b/lib/api/objectGetLegalHold.js index 8940c2ba8c..e815493333 100644 --- a/lib/api/objectGetLegalHold.js +++ b/lib/api/objectGetLegalHold.js @@ -1,8 +1,7 @@ const { promisify } = require('util'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); diff --git a/lib/api/objectGetRetention.js b/lib/api/objectGetRetention.js index 3ac4c19c98..6756be4026 100644 --- a/lib/api/objectGetRetention.js +++ b/lib/api/objectGetRetention.js @@ -1,8 +1,7 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -44,71 +43,73 @@ function objectGetRetention(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectGetRetention', error: err }); - return next(err); + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectGetRetention', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectGetRetention', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + if (reqVersionId) { + log.trace('requested version is delete marker', { method: 'objectGetRetention' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed); + } + log.trace('most recent version is delete marker', { method: 'objectGetRetention' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.NoSuchKey); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectGetRetention' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + ); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + const { retentionMode, retentionDate } = objectMD; + if (!retentionMode || !retentionDate) { + return next(errors.NoSuchObjectLockConfiguration); } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectGetRetention', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - if (reqVersionId) { - log.trace('requested version is delete marker', - { method: 'objectGetRetention' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed); - } - log.trace('most recent version is delete marker', - { method: 'objectGetRetention' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.NoSuchKey); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectGetRetention' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration')); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - const { retentionMode, retentionDate } = objectMD; - if (!retentionMode || !retentionDate) { - return next(errors.NoSuchObjectLockConfiguration); + const xml = convertToXml(retentionMode, retentionDate); + return next(null, bucket, xml, objectMD); + }, + ], + (err, bucket, xml, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectGetRetention' }); + } else { + pushMetric('getObjectRetention', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - const xml = convertToXml(retentionMode, retentionDate); - return next(null, bucket, xml, objectMD); + return callback(err, xml, additionalResHeaders); }, - ], (err, bucket, xml, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectGetRetention' }); - } else { - pushMetric('getObjectRetention', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, xml, additionalResHeaders); - }); + ); } module.exports = objectGetRetention; diff --git a/lib/api/objectGetTagging.js b/lib/api/objectGetTagging.js index 9d52b528a7..4a743b24fc 100644 --- a/lib/api/objectGetTagging.js +++ b/lib/api/objectGetTagging.js @@ -1,8 +1,7 @@ const async = require('async'); const { errors, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader } - = require('./apiUtils/object/versioning'); +const { decodeVersionId, getVersionIdResHeader } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -45,66 +44,64 @@ function objectGetTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectGetTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectGetTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - if (reqVersionId) { - log.trace('requested version is delete marker', - { method: 'objectGetTagging' }); - return next(errors.MethodNotAllowed); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - } - log.trace('most recent version is delete marker', - { method: 'objectGetTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.NoSuchKey); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - const tags = objectMD.tags; - const xml = convertToXml(tags); - next(null, bucket, xml, objectMD); + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectGetTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectGetTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + if (reqVersionId) { + log.trace('requested version is delete marker', { method: 'objectGetTagging' }); + return next(errors.MethodNotAllowed); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + } + log.trace('most recent version is delete marker', { method: 'objectGetTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.NoSuchKey); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + const tags = objectMD.tags; + const xml = convertToXml(tags); + next(null, bucket, xml, objectMD); + }, + ], + (err, bucket, xml, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectGetTagging' }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObjectTagging'); + } else { + pushMetric('getObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('GET', bucketName, '200', 'getObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); + } + return callback(err, xml, additionalResHeaders); }, - ], (err, bucket, xml, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectGetTagging' }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObjectTagging'); - } else { - pushMetric('getObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'GET', bucketName, '200', 'getObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, xml, additionalResHeaders); - }); + ); } module.exports = objectGetTagging; diff --git a/lib/api/objectHead.js b/lib/api/objectHead.js index 4ca8786a8c..e34a7c8205 100644 --- a/lib/api/objectHead.js +++ b/lib/api/objectHead.js @@ -8,8 +8,7 @@ const collectResponseHeaders = require('../utilities/collectResponseHeaders'); const { pushMetric } = require('../utapi/utilities'); const { getVersionIdResHeader } = require('./apiUtils/object/versioning'); const monitoring = require('../utilities/monitoringHandler'); -const { getPartNumber, getPartSize, getPartCountFromMd5 } = - require('./apiUtils/object/partInfo'); +const { getPartNumber, getPartSize, getPartCountFromMd5 } = require('./apiUtils/object/partInfo'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { maximumAllowedPartCount } = require('../../constants'); @@ -58,50 +57,40 @@ function objectHead(authInfo, request, log, callback) { request, }; - return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, - (err, bucket, objMD) => updateEncryption(err, bucket, objMD, objectKey, log, {}, - (err, bucket, objMD) => { - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + return standardMetadataValidateBucketAndObj(mdValParams, request.actionImplicitDenies, log, (err, bucket, objMD) => + updateEncryption(err, bucket, objMD, objectKey, log, {}, (err, bucket, objMD) => { + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { log.debug('error validating request', { error: err, method: 'objectHead', }); - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headObject'); return callback(err, corsHeaders); } if (!objMD) { const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headObject'); return callback(err, corsHeaders); } const verCfg = bucket.getVersioningConfiguration(); if (objMD.isDeleteMarker) { - const responseHeaders = Object.assign({}, - { 'x-amz-delete-marker': true }, corsHeaders); + const responseHeaders = Object.assign({}, { 'x-amz-delete-marker': true }, corsHeaders); if (!versionId) { - monitoring.promMetrics( - 'HEAD', bucketName, 404, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, 404, 'headObject'); return callback(errors.NoSuchKey, responseHeaders); } // return MethodNotAllowed if requesting a specific // version that has a delete marker - responseHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objMD); - monitoring.promMetrics( - 'HEAD', bucketName, 405, 'headObject'); + responseHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objMD); + monitoring.promMetrics('HEAD', bucketName, 405, 'headObject'); return callback(errors.MethodNotAllowed, responseHeaders); } - const headerValResult = validateHeaders(request.headers, - objMD['last-modified'], objMD['content-md5']); + const headerValResult = validateHeaders(request.headers, objMD['last-modified'], objMD['content-md5']); if (headerValResult.error) { return callback(headerValResult.error, corsHeaders); } - const responseHeaders = collectResponseHeaders(objMD, corsHeaders, - verCfg); + const responseHeaders = collectResponseHeaders(objMD, corsHeaders, verCfg); const partNumber = getPartNumber(request.query); // Checksum is not returned for partNumber requests because per-part @@ -109,8 +98,7 @@ function objectHead(authInfo, request, log, callback) { if (checksumMode === 'ENABLED' && !partNumber) { const checksum = objMD.checksum; if (checksum) { - responseHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`] - = checksum.checksumValue; + responseHeaders[`x-amz-checksum-${checksum.checksumAlgorithm}`] = checksum.checksumValue; responseHeaders['x-amz-checksum-type'] = checksum.checksumType; } } @@ -129,8 +117,7 @@ function objectHead(authInfo, request, log, callback) { Object.assign(responseHeaders, setArchiveInfoHeaders(objMD)); } - const objLength = (objMD.location === null ? - 0 : parseInt(objMD['content-length'], 10)); + const objLength = objMD.location === null ? 0 : parseInt(objMD['content-length'], 10); // Store full object size for server access logs if (request.serverAccessLog) { // eslint-disable-next-line no-param-reassign @@ -139,30 +126,26 @@ function objectHead(authInfo, request, log, callback) { let byteRange; if (request.headers.range) { - const { range, error } - = parseRange(request.headers.range, objLength); + const { range, error } = parseRange(request.headers.range, objLength); if (error) { return callback(error, corsHeaders); } responseHeaders['accept-ranges'] = 'bytes'; if (range) { byteRange = range; - responseHeaders['content-length'] = - range[1] - range[0] + 1; - responseHeaders['content-range'] = - `bytes ${range[0]}-${range[1]}/${objLength}`; + responseHeaders['content-length'] = range[1] - range[0] + 1; + responseHeaders['content-range'] = `bytes ${range[0]}-${range[1]}/${objLength}`; } } if (partNumber !== undefined) { if (byteRange) { - const error = errorInstances.InvalidRequest - .customizeDescription('Cannot specify both Range ' + - 'header and partNumber query parameter.'); + const error = errorInstances.InvalidRequest.customizeDescription( + 'Cannot specify both Range ' + 'header and partNumber query parameter.', + ); return callback(error, corsHeaders); } if (Number.isNaN(partNumber)) { - const error = errorInstances.InvalidArgument - .customizeDescription('Part number must be a number.'); + const error = errorInstances.InvalidArgument.customizeDescription('Part number must be a number.'); return callback(error, corsHeaders); } if (partNumber < 1 || partNumber > maximumAllowedPartCount) { @@ -191,7 +174,8 @@ function objectHead(authInfo, request, log, callback) { }); monitoring.promMetrics('HEAD', bucketName, '200', 'headObject'); return callback(null, responseHeaders); - })); + }), + ); } module.exports = objectHead; diff --git a/lib/api/objectPutACL.js b/lib/api/objectPutACL.js index 62045f7cc8..a53c8314ce 100644 --- a/lib/api/objectPutACL.js +++ b/lib/api/objectPutACL.js @@ -7,8 +7,11 @@ const { pushMetric } = require('../utapi/utilities'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const vault = require('../auth/vault'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } - = require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const monitoring = require('../utilities/monitoringHandler'); const { config } = require('../Config'); @@ -65,11 +68,7 @@ function objectPutACL(authInfo, request, log, cb) { monitoring.promMetrics('PUT', bucketName, 400, 'putObjectAcl'); return cb(errors.InvalidArgument); } - const possibleGroups = [ - constants.publicId, - constants.allAuthedUsersId, - constants.logId, - ]; + const possibleGroups = [constants.publicId, constants.allAuthedUsersId, constants.logId]; const decodedVidResult = decodeVersionId(request.query); if (decodedVidResult instanceof Error) { @@ -100,224 +99,217 @@ function objectPutACL(authInfo, request, log, cb) { READ_ACP: [], }; - const grantReadHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-read'], 'READ'); - const grantReadACPHeader = - aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], - 'READ_ACP'); - const grantWriteACPHeader = aclUtils.parseGrant( - request.headers['x-amz-grant-write-acp'], 'WRITE_ACP'); - const grantFullControlHeader = aclUtils.parseGrant( - request.headers['x-amz-grant-full-control'], 'FULL_CONTROL'); + const grantReadHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read'], 'READ'); + const grantReadACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(request.headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(request.headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - return async.waterfall([ - function validateBucketAndObj(next) { - return standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('delete marker detected', - { method: 'objectPutACL' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); + return async.waterfall( + [ + function validateBucketAndObj(next) { + return standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('delete marker detected', { method: 'objectPutACL' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ); + }, + function parseAclFromXml(bucket, objectMD, next) { + // If not setting acl through headers, parse body + let jsonGrants; + let aclOwnerID; + if ( + newCannedACL === undefined && + grantReadHeader === undefined && + grantReadACPHeader === undefined && + grantWriteACPHeader === undefined && + grantFullControlHeader === undefined + ) { + if (request.post) { + log.trace('using acls from request body'); + return aclUtils.parseAclXml(request.post, log, (err, jsonGrants, aclOwnerID) => + next(err, bucket, objectMD, jsonGrants, aclOwnerID), + ); } - return next(null, bucket, objectMD); - }); - }, - function parseAclFromXml(bucket, objectMD, next) { - // If not setting acl through headers, parse body - let jsonGrants; - let aclOwnerID; - if (newCannedACL === undefined - && grantReadHeader === undefined - && grantReadACPHeader === undefined - && grantWriteACPHeader === undefined - && grantFullControlHeader === undefined) { - if (request.post) { - log.trace('using acls from request body'); - return aclUtils.parseAclXml(request.post, log, - (err, jsonGrants, aclOwnerID) => next(err, bucket, - objectMD, jsonGrants, aclOwnerID)); + // If no ACLs sent with request at all + return next(errors.MalformedXML, bucket); } - // If no ACLs sent with request at all - return next(errors.MalformedXML, bucket); - } - /** - * If acl set in headers (including canned acl) pass bucket and - * undefined to the next function - */ - log.debug('using acls from request headers'); - return next(null, bucket, objectMD, jsonGrants, aclOwnerID); - }, - function processAcls(bucket, objectMD, jsonGrants, aclOwnerID, next) { - if (newCannedACL) { - log.debug('canned acl', { cannedAcl: newCannedACL }); - addACLParams.Canned = newCannedACL; - return next(null, bucket, objectMD, addACLParams); - } - let usersIdentifiedByEmail = []; - let usersIdentifiedByGroup = []; - let usersIdentifiedByID = []; - let hasError = false; + /** + * If acl set in headers (including canned acl) pass bucket and + * undefined to the next function + */ + log.debug('using acls from request headers'); + return next(null, bucket, objectMD, jsonGrants, aclOwnerID); + }, + function processAcls(bucket, objectMD, jsonGrants, aclOwnerID, next) { + if (newCannedACL) { + log.debug('canned acl', { cannedAcl: newCannedACL }); + addACLParams.Canned = newCannedACL; + return next(null, bucket, objectMD, addACLParams); + } + let usersIdentifiedByEmail = []; + let usersIdentifiedByGroup = []; + let usersIdentifiedByID = []; + let hasError = false; - // If grants set by xml and xml owner ID is incorrect - if (aclOwnerID && (aclOwnerID !== objectMD['owner-id'])) { - log.trace('incorrect owner ID provided in ACL', { - ACL: request.post, - method: 'objectPutACL', - }); - return next(errors.AccessDenied, bucket); - } + // If grants set by xml and xml owner ID is incorrect + if (aclOwnerID && aclOwnerID !== objectMD['owner-id']) { + log.trace('incorrect owner ID provided in ACL', { + ACL: request.post, + method: 'objectPutACL', + }); + return next(errors.AccessDenied, bucket); + } - /** - * If grants set by xml, loop through the grants - * and separate grant types so parsed in same manner - * as header grants - */ - if (jsonGrants) { - log.trace('parsing acl grants'); - jsonGrants.forEach(grant => { - const grantee = grant.Grantee[0]; - const granteeType = grantee.$['xsi:type']; - const permission = grant.Permission[0]; - let skip = false; - if (possibleGrants.indexOf(permission) < 0) { - skip = true; - } - if (!skip && granteeType === 'AmazonCustomerByEmail') { - usersIdentifiedByEmail.push({ - identifier: grantee.EmailAddress[0], - grantType: permission, - userIDType: 'emailaddress', - }); - } - if (!skip && granteeType === 'CanonicalUser') { - usersIdentifiedByID.push({ - identifier: grantee.ID[0], - grantType: permission, - userIDType: 'id', - }); - } - if (!skip && granteeType === 'Group') { - if (possibleGroups.indexOf(grantee.URI[0]) < 0) { - log.trace('invalid user group', - { userGroup: grantee.URI[0] }); - hasError = true; - return next(errors.InvalidArgument, bucket); + /** + * If grants set by xml, loop through the grants + * and separate grant types so parsed in same manner + * as header grants + */ + if (jsonGrants) { + log.trace('parsing acl grants'); + jsonGrants.forEach(grant => { + const grantee = grant.Grantee[0]; + const granteeType = grantee.$['xsi:type']; + const permission = grant.Permission[0]; + let skip = false; + if (possibleGrants.indexOf(permission) < 0) { + skip = true; + } + if (!skip && granteeType === 'AmazonCustomerByEmail') { + usersIdentifiedByEmail.push({ + identifier: grantee.EmailAddress[0], + grantType: permission, + userIDType: 'emailaddress', + }); + } + if (!skip && granteeType === 'CanonicalUser') { + usersIdentifiedByID.push({ + identifier: grantee.ID[0], + grantType: permission, + userIDType: 'id', + }); + } + if (!skip && granteeType === 'Group') { + if (possibleGroups.indexOf(grantee.URI[0]) < 0) { + log.trace('invalid user group', { userGroup: grantee.URI[0] }); + hasError = true; + return next(errors.InvalidArgument, bucket); + } + return usersIdentifiedByGroup.push({ + identifier: grantee.URI[0], + grantType: permission, + userIDType: 'uri', + }); } - return usersIdentifiedByGroup.push({ - identifier: grantee.URI[0], - grantType: permission, - userIDType: 'uri', - }); + return undefined; + }); + if (hasError) { + return undefined; } - return undefined; - }); - if (hasError) { - return undefined; - } - } else { - // If no canned ACL and no parsed xml, loop - // through the access headers - const allGrantHeaders = - [].concat(grantReadHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader); + } else { + // If no canned ACL and no parsed xml, loop + // through the access headers + const allGrantHeaders = [].concat( + grantReadHeader, + grantReadACPHeader, + grantWriteACPHeader, + grantFullControlHeader, + ); - usersIdentifiedByEmail = allGrantHeaders.filter(item => - item && item.userIDType.toLowerCase() === 'emailaddress'); - usersIdentifiedByGroup = allGrantHeaders - .filter(itm => itm && itm.userIDType - .toLowerCase() === 'uri'); - for (let i = 0; i < usersIdentifiedByGroup.length; i++) { - if (possibleGroups.indexOf( - usersIdentifiedByGroup[i].identifier) < 0) { - log.trace('invalid user group', - { userGroup: usersIdentifiedByGroup[i] - .identifier }); - return next(errors.InvalidArgument, bucket); + usersIdentifiedByEmail = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'emailaddress', + ); + usersIdentifiedByGroup = allGrantHeaders.filter( + itm => itm && itm.userIDType.toLowerCase() === 'uri', + ); + for (let i = 0; i < usersIdentifiedByGroup.length; i++) { + if (possibleGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) { + log.trace('invalid user group', { userGroup: usersIdentifiedByGroup[i].identifier }); + return next(errors.InvalidArgument, bucket); + } } + /** TODO: Consider whether want to verify with Vault + * whether canonicalID is associated with existing + * account before adding to ACL */ + usersIdentifiedByID = allGrantHeaders.filter( + item => item && item.userIDType.toLowerCase() === 'id', + ); } - /** TODO: Consider whether want to verify with Vault - * whether canonicalID is associated with existing - * account before adding to ACL */ - usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType - .toLowerCase() === 'id'); - } - const justEmails = usersIdentifiedByEmail - .map(item => item.identifier); - // If have to lookup canonicalID's do that asynchronously - if (justEmails.length > 0) { - return vault.getCanonicalIds( - justEmails, log, (err, results) => { + const justEmails = usersIdentifiedByEmail.map(item => item.identifier); + // If have to lookup canonicalID's do that asynchronously + if (justEmails.length > 0) { + return vault.getCanonicalIds(justEmails, log, (err, results) => { if (err) { - log.trace('error looking up canonical ids', - { error: err, method: 'getCanonicalIDs' }); + log.trace('error looking up canonical ids', { error: err, method: 'getCanonicalIDs' }); return next(err, bucket); } - const reconstructedUsersIdentifiedByEmail = aclUtils - .reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = aclUtils - .sortHeaderGrants(allUsers, addACLParams); - return next(null, bucket, objectMD, - revisedAddACLParams); + usersIdentifiedByGroup, + ); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); + return next(null, bucket, objectMD, revisedAddACLParams); }); + } + const allUsers = [].concat(usersIdentifiedByID, usersIdentifiedByGroup); + const revisedAddACLParams = aclUtils.sortHeaderGrants(allUsers, addACLParams); + return next(null, bucket, objectMD, revisedAddACLParams); + }, + function addAclsToObjMD(bucket, objectMD, ACLParams, next) { + // Add acl's to object metadata + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + acl.addObjectACL(bucket, objectKey, objectMD, ACLParams, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], + (err, bucket, objectMD) => { + const resHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { + error: err, + method: 'objectPutACL', + }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectAcl'); + return cb(err, resHeaders); } - const allUsers = [].concat( - usersIdentifiedByID, - usersIdentifiedByGroup); - const revisedAddACLParams = - aclUtils.sortHeaderGrants(allUsers, addACLParams); - return next(null, bucket, objectMD, revisedAddACLParams); - }, - function addAclsToObjMD(bucket, objectMD, ACLParams, next) { - // Add acl's to object metadata - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - acl.addObjectACL(bucket, objectKey, objectMD, - ACLParams, params, log, err => next(err, bucket, objectMD)); - }, - ], (err, bucket, objectMD) => { - const resHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { - error: err, - method: 'objectPutACL', - }); - monitoring.promMetrics( - 'PUT', bucketName, err.code, 'putObjectAcl'); - return cb(err, resHeaders); - } - const verCfg = bucket.getVersioningConfiguration(); - resHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); + const verCfg = bucket.getVersioningConfiguration(); + resHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); - log.trace('processed request successfully in object put acl api'); - pushMetric('putObjectAcl', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics('PUT', bucketName, '200', 'putObjectAcl'); - return cb(null, resHeaders); - }); + log.trace('processed request successfully in object put acl api'); + pushMetric('putObjectAcl', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectAcl'); + return cb(null, resHeaders); + }, + ); } module.exports = objectPutACL; diff --git a/lib/api/objectRestore.js b/lib/api/objectRestore.js index be0eb48389..f96234dc4e 100644 --- a/lib/api/objectRestore.js +++ b/lib/api/objectRestore.js @@ -22,8 +22,7 @@ const sdtObjectRestore = require('./apiUtils/object/objectRestore'); * @return {undefined} */ function objectRestore(userInfo, request, log, callback) { - return sdtObjectRestore(metadata, metadataUtils, userInfo, request, - log, callback); + return sdtObjectRestore(metadata, metadataUtils, userInfo, request, log, callback); } module.exports = objectRestore; diff --git a/lib/api/serviceGet.js b/lib/api/serviceGet.js index f81169b3f9..79b980b8c0 100644 --- a/lib/api/serviceGet.js +++ b/lib/api/serviceGet.js @@ -34,9 +34,8 @@ function generateXml(xml, owner, userBuckets, splitter) { xml.push( '', `${key}`, - `${bucket.value.creationDate}` + - '', - '' + `${bucket.value.creationDate}` + '', + '', ); }); xml.push(''); @@ -56,38 +55,32 @@ function serviceGet(authInfo, request, log, callback) { if (authInfo.isRequesterPublicUser()) { log.debug('operation not available for public user'); - monitoring.promMetrics( - 'GET', request.bucketName, 403, 'getService'); + monitoring.promMetrics('GET', request.bucketName, 403, 'getService'); return callback(errors.AccessDenied); } const xml = []; const canonicalId = authInfo.getCanonicalID(); xml.push( '', - '', + '', '', `${canonicalId}`, - `${authInfo.getAccountDisplayName()}` + - '', + `${authInfo.getAccountDisplayName()}` + '', '', - '' + '', ); - return services.getService(authInfo, request, log, constants.splitter, - (err, userBuckets, splitter) => { - if (err) { - monitoring.promMetrics( - 'GET', userBuckets, err.code, 'getService'); - return callback(err); - } - // TODO push metric for serviceGet - // pushMetric('getService', log, { - // bucket: bucketName, - // }); - monitoring.promMetrics('GET', userBuckets, '200', 'getService'); - return callback(null, generateXml(xml, canonicalId, userBuckets, - splitter)); - }); + return services.getService(authInfo, request, log, constants.splitter, (err, userBuckets, splitter) => { + if (err) { + monitoring.promMetrics('GET', userBuckets, err.code, 'getService'); + return callback(err); + } + // TODO push metric for serviceGet + // pushMetric('getService', log, { + // bucket: bucketName, + // }); + monitoring.promMetrics('GET', userBuckets, '200', 'getService'); + return callback(null, generateXml(xml, canonicalId, userBuckets, splitter)); + }); } module.exports = serviceGet; diff --git a/lib/api/website.js b/lib/api/website.js index 801d92f708..ed23ba3e5e 100644 --- a/lib/api/website.js +++ b/lib/api/website.js @@ -5,10 +5,12 @@ const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const constants = require('../../constants'); const metadata = require('../metadata/wrapper'); const bucketShield = require('./apiUtils/bucket/bucketShield'); -const { appendWebsiteIndexDocument, findRoutingRule, extractRedirectInfo } = - require('./apiUtils/object/websiteServing'); -const { isObjAuthorized, isBucketAuthorized } = - require('./apiUtils/authorization/permissionChecks'); +const { + appendWebsiteIndexDocument, + findRoutingRule, + extractRedirectInfo, +} = require('./apiUtils/object/websiteServing'); +const { isObjAuthorized, isBucketAuthorized } = require('./apiUtils/authorization/permissionChecks'); const collectResponseHeaders = require('../utilities/collectResponseHeaders'); const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); @@ -27,81 +29,74 @@ const monitoring = require('../utilities/monitoringHandler'); * @param {function} callback - callback to function in route * @return {undefined} */ -function _errorActions(err, errorDocument, routingRules, - bucket, objectKey, corsHeaders, request, log, callback) { +function _errorActions(err, errorDocument, routingRules, bucket, objectKey, corsHeaders, request, log, callback) { const bucketName = bucket.getName(); - const errRoutingRule = findRoutingRule(routingRules, - objectKey, err.code); + const errRoutingRule = findRoutingRule(routingRules, objectKey, err.code); if (errRoutingRule) { // route will redirect const action = request.method === 'HEAD' ? 'headObject' : 'getObject'; - monitoring.promMetrics( - request.method, bucketName, err.code, action); - return callback(err, false, null, corsHeaders, errRoutingRule, - objectKey); + monitoring.promMetrics(request.method, bucketName, err.code, action); + return callback(err, false, null, corsHeaders, errRoutingRule, objectKey); } if (request.method === 'HEAD') { - monitoring.promMetrics( - 'HEAD', bucketName, err.code, 'headObject'); + monitoring.promMetrics('HEAD', bucketName, err.code, 'headObject'); return callback(err, false, null, corsHeaders); } if (errorDocument) { - return metadata.getObjectMD(bucketName, errorDocument, {}, log, - (errObjErr, errObjMD) => { - if (errObjErr) { - // error retrieving error document so return original error - // and set boolean of error retrieving user's error document - // to true - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, true, null, corsHeaders); - } - // return the default error message if the object is private - // rather than sending a stored error file - // eslint-disable-next-line no-param-reassign - request.objectKey = errorDocument; - if (!isObjAuthorized(bucket, errObjMD, request.apiMethods || 'objectGet', - constants.publicId, null, log, request, request.actionImplicitDenies, true)) { - log.trace('errorObj not authorized', { error: err }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, true, null, corsHeaders); - } - const dataLocator = errObjMD.location; - if (errObjMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - errObjMD['x-amz-server-side-encryption-aws-' + - 'kms-key-id']; - dataLocator[i].algorithm = - errObjMD['x-amz-server-side-encryption']; - } + return metadata.getObjectMD(bucketName, errorDocument, {}, log, (errObjErr, errObjMD) => { + if (errObjErr) { + // error retrieving error document so return original error + // and set boolean of error retrieving user's error document + // to true + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, true, null, corsHeaders); + } + // return the default error message if the object is private + // rather than sending a stored error file + // eslint-disable-next-line no-param-reassign + request.objectKey = errorDocument; + if ( + !isObjAuthorized( + bucket, + errObjMD, + request.apiMethods || 'objectGet', + constants.publicId, + null, + log, + request, + request.actionImplicitDenies, + true, + ) + ) { + log.trace('errorObj not authorized', { error: err }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, true, null, corsHeaders); + } + const dataLocator = errObjMD.location; + if (errObjMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = errObjMD['x-amz-server-side-encryption-aws-' + 'kms-key-id']; + dataLocator[i].algorithm = errObjMD['x-amz-server-side-encryption']; } + } - if (errObjMD['x-amz-website-redirect-location']) { - const redirectLocation = - errObjMD['x-amz-website-redirect-location']; - const redirectInfo = { withError: true, - location: redirectLocation }; - log.trace('redirecting to x-amz-website-redirect-location', - { location: redirectLocation }); - return callback(err, false, dataLocator, corsHeaders, - redirectInfo, ''); - } + if (errObjMD['x-amz-website-redirect-location']) { + const redirectLocation = errObjMD['x-amz-website-redirect-location']; + const redirectInfo = { withError: true, location: redirectLocation }; + log.trace('redirecting to x-amz-website-redirect-location', { location: redirectLocation }); + return callback(err, false, dataLocator, corsHeaders, redirectInfo, ''); + } - const responseMetaHeaders = collectResponseHeaders(errObjMD, - corsHeaders); - pushMetric('getObject', log, { - bucket: bucketName, - newByteLength: responseMetaHeaders['Content-Length'], - }); - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); - return callback(err, false, dataLocator, responseMetaHeaders); + const responseMetaHeaders = collectResponseHeaders(errObjMD, corsHeaders); + pushMetric('getObject', log, { + bucket: bucketName, + newByteLength: responseMetaHeaders['Content-Length'], }); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); + return callback(err, false, dataLocator, responseMetaHeaders); + }); } - monitoring.promMetrics( - 'GET', bucketName, err.code, 'getObject'); + monitoring.promMetrics('GET', bucketName, err.code, 'getObject'); return callback(err, false, null, corsHeaders); } @@ -120,8 +115,7 @@ function capitalize(str) { * @returns {function} HEAD callback with GET signature */ function callbackGetToHead(callback) { - return (err, userErrorPageFailure, dataGetInfo, - resMetaHeaders, redirectInfo, key) => + return (err, userErrorPageFailure, dataGetInfo, resMetaHeaders, redirectInfo, key) => callback(err, resMetaHeaders, redirectInfo, key); } @@ -147,26 +141,21 @@ function website(request, log, callback) { return metadata.getBucket(bucketName, log, (err, bucket) => { if (err) { log.trace('error retrieving bucket metadata', { error: err }); - monitoring.promMetrics( - request.method, bucketName, err.code, action); + monitoring.promMetrics(request.method, bucketName, err.code, action); return callback(err, false); } if (bucketShield(bucket, `object${methodCapitalized}`)) { log.trace('bucket in transient/deleted state so shielding'); - monitoring.promMetrics( - request.method, bucketName, 404, action); + monitoring.promMetrics(request.method, bucketName, 404, action); return callback(errors.NoSuchBucket, false); } - const corsHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); + const corsHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); // bucket ACL's do not matter for website head since it is always the // head of an object. object ACL's are what matter const websiteConfig = bucket.getWebsiteConfiguration(); if (!websiteConfig) { - monitoring.promMetrics( - request.method, bucketName, 404, action); - return callback(errors.NoSuchWebsiteConfiguration, false, null, - corsHeaders); + monitoring.promMetrics(request.method, bucketName, 404, action); + return callback(errors.NoSuchWebsiteConfiguration, false, null, corsHeaders); } // any errors above would be our own created generic error html // if have a website config, error going forward would be user's @@ -174,8 +163,7 @@ function website(request, log, callback) { // handle redirect all if (websiteConfig.getRedirectAllRequestsTo()) { - return callback(null, false, null, corsHeaders, - websiteConfig.getRedirectAllRequestsTo(), reqObjectKey); + return callback(null, false, null, corsHeaders, websiteConfig.getRedirectAllRequestsTo(), reqObjectKey); } // check whether need to redirect based on key @@ -185,8 +173,7 @@ function website(request, log, callback) { if (keyRoutingRule) { // TODO: optimize by not rerouting if only routing // rule is to change out key - return callback(null, false, null, corsHeaders, - keyRoutingRule, reqObjectKey); + return callback(null, false, null, corsHeaders, keyRoutingRule, reqObjectKey); } appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument()); @@ -203,107 +190,134 @@ function website(request, log, callback) { function runWebsite(originalError) { // get object metadata and check authorization and header // validation - return metadata.getObjectMD(bucketName, request.objectKey, {}, log, - (err, objMD) => { - // Note: In case of error, we intentionally send the original - // object key to _errorActions as in case of a redirect, we do - // not want to append index key to redirect location - if (err) { - log.trace('error retrieving object metadata', - { error: err }); - let returnErr = err; - const bucketAuthorized = isBucketAuthorized(bucket, request.apiMethods || 'bucketGet', - constants.publicId, null, log, request, request.actionImplicitDenies, true); - // if index object does not exist and bucket is private AWS - // returns 403 - AccessDenied error. - if (err.is.NoSuchKey && !bucketAuthorized) { - returnErr = errors.AccessDenied; - } - - // Check if key is a folder containing index for redirect 302 - // https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html - if (!originalError && reqObjectKey && !reqObjectKey.endsWith('/')) { - appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument(), true); - // propagate returnErr as originalError to be used if index is not found - return runWebsite(returnErr); - } - - return _errorActions(originalError || returnErr, - websiteConfig.getErrorDocument(), routingRules, - bucket, reqObjectKey, corsHeaders, request, log, - callback); - } - if (!isObjAuthorized(bucket, objMD, request.apiMethods || 'objectGet', - constants.publicId, null, log, request, request.actionImplicitDenies, true)) { - const err = errors.AccessDenied; - log.trace('request not authorized', { error: err }); - return _errorActions(err, websiteConfig.getErrorDocument(), - routingRules, bucket, - reqObjectKey, corsHeaders, request, log, callback); + return metadata.getObjectMD(bucketName, request.objectKey, {}, log, (err, objMD) => { + // Note: In case of error, we intentionally send the original + // object key to _errorActions as in case of a redirect, we do + // not want to append index key to redirect location + if (err) { + log.trace('error retrieving object metadata', { error: err }); + let returnErr = err; + const bucketAuthorized = isBucketAuthorized( + bucket, + request.apiMethods || 'bucketGet', + constants.publicId, + null, + log, + request, + request.actionImplicitDenies, + true, + ); + // if index object does not exist and bucket is private AWS + // returns 403 - AccessDenied error. + if (err.is.NoSuchKey && !bucketAuthorized) { + returnErr = errors.AccessDenied; } - // access granted to index document, needs a redirect 302 - // to the original key with trailing / - if (originalError) { - const redirectInfo = { withError: true, - location: `/${reqObjectKey}/` }; - return callback(errors.Found, false, null, corsHeaders, - redirectInfo, ''); + // Check if key is a folder containing index for redirect 302 + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/IndexDocumentSupport.html + if (!originalError && reqObjectKey && !reqObjectKey.endsWith('/')) { + appendWebsiteIndexDocument(request, websiteConfig.getIndexDocument(), true); + // propagate returnErr as originalError to be used if index is not found + return runWebsite(returnErr); } - const headerValResult = validateHeaders(request.headers, - objMD['last-modified'], objMD['content-md5']); - if (headerValResult.error) { - const err = headerValResult.error; - log.trace('header validation error', { error: err }); - return _errorActions(err, websiteConfig.getErrorDocument(), - routingRules, bucket, reqObjectKey, - corsHeaders, request, log, callback); - } - // check if object to serve has website redirect header - // Note: AWS prioritizes website configuration rules over - // object key's website redirect header, so we make the - // check at the end. - if (objMD['x-amz-website-redirect-location']) { - const redirectLocation = - objMD['x-amz-website-redirect-location']; - const redirectInfo = - extractRedirectInfo(redirectLocation); - log.trace('redirecting to x-amz-website-redirect-location', - { location: redirectLocation }); - return callback(null, false, null, corsHeaders, - redirectInfo, ''); - } - // got obj metadata, authorized and headers validated, - // good to go - const responseMetaHeaders = collectResponseHeaders(objMD, - corsHeaders); + return _errorActions( + originalError || returnErr, + websiteConfig.getErrorDocument(), + routingRules, + bucket, + reqObjectKey, + corsHeaders, + request, + log, + callback, + ); + } + if ( + !isObjAuthorized( + bucket, + objMD, + request.apiMethods || 'objectGet', + constants.publicId, + null, + log, + request, + request.actionImplicitDenies, + true, + ) + ) { + const err = errors.AccessDenied; + log.trace('request not authorized', { error: err }); + return _errorActions( + err, + websiteConfig.getErrorDocument(), + routingRules, + bucket, + reqObjectKey, + corsHeaders, + request, + log, + callback, + ); + } - if (request.method === 'HEAD') { - pushMetric('headObject', log, { bucket: bucketName }); - monitoring.promMetrics('HEAD', - bucketName, '200', 'headObject'); - return callback(null, false, null, responseMetaHeaders); - } + // access granted to index document, needs a redirect 302 + // to the original key with trailing / + if (originalError) { + const redirectInfo = { withError: true, location: `/${reqObjectKey}/` }; + return callback(errors.Found, false, null, corsHeaders, redirectInfo, ''); + } + + const headerValResult = validateHeaders(request.headers, objMD['last-modified'], objMD['content-md5']); + if (headerValResult.error) { + const err = headerValResult.error; + log.trace('header validation error', { error: err }); + return _errorActions( + err, + websiteConfig.getErrorDocument(), + routingRules, + bucket, + reqObjectKey, + corsHeaders, + request, + log, + callback, + ); + } + // check if object to serve has website redirect header + // Note: AWS prioritizes website configuration rules over + // object key's website redirect header, so we make the + // check at the end. + if (objMD['x-amz-website-redirect-location']) { + const redirectLocation = objMD['x-amz-website-redirect-location']; + const redirectInfo = extractRedirectInfo(redirectLocation); + log.trace('redirecting to x-amz-website-redirect-location', { location: redirectLocation }); + return callback(null, false, null, corsHeaders, redirectInfo, ''); + } + // got obj metadata, authorized and headers validated, + // good to go + const responseMetaHeaders = collectResponseHeaders(objMD, corsHeaders); + + if (request.method === 'HEAD') { + pushMetric('headObject', log, { bucket: bucketName }); + monitoring.promMetrics('HEAD', bucketName, '200', 'headObject'); + return callback(null, false, null, responseMetaHeaders); + } - const dataLocator = objMD.location; - if (objMD['x-amz-server-side-encryption']) { - for (let i = 0; i < dataLocator.length; i++) { - dataLocator[i].masterKeyId = - objMD['x-amz-server-side-encryption-aws-' + - 'kms-key-id']; - dataLocator[i].algorithm = - objMD['x-amz-server-side-encryption']; - } + const dataLocator = objMD.location; + if (objMD['x-amz-server-side-encryption']) { + for (let i = 0; i < dataLocator.length; i++) { + dataLocator[i].masterKeyId = objMD['x-amz-server-side-encryption-aws-' + 'kms-key-id']; + dataLocator[i].algorithm = objMD['x-amz-server-side-encryption']; } - pushMetric('getObject', log, { - bucket: bucketName, - newByteLength: responseMetaHeaders['Content-Length'], - }); - monitoring.promMetrics('GET', bucketName, '200', - 'getObject', responseMetaHeaders['Content-Length']); - return callback(null, false, dataLocator, responseMetaHeaders); + } + pushMetric('getObject', log, { + bucket: bucketName, + newByteLength: responseMetaHeaders['Content-Length'], }); + monitoring.promMetrics('GET', bucketName, '200', 'getObject', responseMetaHeaders['Content-Length']); + return callback(null, false, dataLocator, responseMetaHeaders); + }); } return runWebsite(); diff --git a/lib/auth/in_memory/builder.js b/lib/auth/in_memory/builder.js index 37b358e6d5..068b58ceb4 100644 --- a/lib/auth/in_memory/builder.js +++ b/lib/auth/in_memory/builder.js @@ -1,5 +1,4 @@ -const serviceAccountPrefix = - require('arsenal').constants.zenkoServiceAccount; +const serviceAccountPrefix = require('arsenal').constants.zenkoServiceAccount; /** build simple authdata with only one account * @param {string} accessKey - account's accessKey @@ -9,26 +8,31 @@ const serviceAccountPrefix = * @param {string} userName - account's user name * @return {object} authdata - authdata with account's accessKey and secretKey */ -function buildAuthDataAccount(accessKey, secretKey, canonicalId, serviceName, -userName) { +function buildAuthDataAccount(accessKey, secretKey, canonicalId, serviceName, userName) { // TODO: remove specific check for clueso and generate unique // canonical id's for accounts - const finalCanonicalId = canonicalId || - (serviceName ? `${serviceAccountPrefix}/${serviceName}` : - '12349df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47qwer'); + const finalCanonicalId = + canonicalId || + (serviceName + ? `${serviceAccountPrefix}/${serviceName}` + : '12349df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47qwer'); const shortid = '123456789012'; return { - accounts: [{ - name: userName || 'CustomAccount', - email: 'customaccount1@setbyenv.com', - arn: `arn:aws:iam::${shortid}:root`, - canonicalID: finalCanonicalId, - shortid, - keys: [{ - access: accessKey, - secret: secretKey, - }], - }], + accounts: [ + { + name: userName || 'CustomAccount', + email: 'customaccount1@setbyenv.com', + arn: `arn:aws:iam::${shortid}:root`, + canonicalID: finalCanonicalId, + shortid, + keys: [ + { + access: accessKey, + secret: secretKey, + }, + ], + }, + ], }; } diff --git a/lib/auth/streamingV4/ChecksumTransform.js b/lib/auth/streamingV4/ChecksumTransform.js index bf36624d83..046ad3ae9a 100644 --- a/lib/auth/streamingV4/ChecksumTransform.js +++ b/lib/auth/streamingV4/ChecksumTransform.js @@ -74,14 +74,16 @@ class ChecksumTransform extends Transform { } _flush(callback) { - Promise.resolve(this.algo.digestFromHash(this.hash)) - .then(digest => { + Promise.resolve(this.algo.digestFromHash(this.hash)).then( + digest => { this.digest = digest; callback(); - }, err => { + }, + err => { this.log.error('failed to compute checksum digest', { error: err, algorithm: this.algoName }); callback(errors.InternalError); - }); + }, + ); } _transform(chunk, encoding, callback) { diff --git a/lib/auth/streamingV4/V4Transform.js b/lib/auth/streamingV4/V4Transform.js index e18d20e14f..57576733a8 100644 --- a/lib/auth/streamingV4/V4Transform.js +++ b/lib/auth/streamingV4/V4Transform.js @@ -28,8 +28,7 @@ class V4Transform extends Transform { * @param {function} errCb - callback called if an error occurs */ constructor(streamingV4Params, log, errCb) { - const { accessKey, signatureFromRequest, region, scopeDate, timestamp, - credentialScope } = streamingV4Params; + const { accessKey, signatureFromRequest, region, scopeDate, timestamp, credentialScope } = streamingV4Params; super({}); this.log = log; this.errCb = errCb; @@ -79,28 +78,24 @@ class V4Transform extends Transform { this.currentMetadata.push(remainingPlusStoredMetadata); return { completeMetadata: false }; } - let fullMetadata = remainingPlusStoredMetadata.slice(0, - lineBreakIndex); + let fullMetadata = remainingPlusStoredMetadata.slice(0, lineBreakIndex); // handle extra line break on end of data chunk if (fullMetadata.length === 0) { - const chunkWithoutLeadingLineBreak = remainingPlusStoredMetadata - .slice(2); + const chunkWithoutLeadingLineBreak = remainingPlusStoredMetadata.slice(2); // find second line break lineBreakIndex = chunkWithoutLeadingLineBreak.indexOf('\r\n'); if (lineBreakIndex < 0) { this.currentMetadata.push(chunkWithoutLeadingLineBreak); return { completeMetadata: false }; } - fullMetadata = chunkWithoutLeadingLineBreak.slice(0, - lineBreakIndex); + fullMetadata = chunkWithoutLeadingLineBreak.slice(0, lineBreakIndex); } const splitMeta = fullMetadata.toString().split(';'); this.log.trace('parsed full metadata for chunk', { splitMeta }); if (splitMeta.length !== 2) { - this.log.trace('chunk body did not contain correct ' + - 'metadata format'); + this.log.trace('chunk body did not contain correct ' + 'metadata format'); return { err: errors.InvalidArgument }; } let dataSize = splitMeta[0]; @@ -132,8 +127,7 @@ class V4Transform extends Transform { completeMetadata: true, // start slice at lineBreak plus 2 to remove line break at end of // metadata piece since length of '\r\n' is 2 - unparsedChunk: remainingPlusStoredMetadata - .slice(lineBreakIndex + 2), + unparsedChunk: remainingPlusStoredMetadata.slice(lineBreakIndex + 2), }; } @@ -146,10 +140,13 @@ class V4Transform extends Transform { */ _authenticate(dataToSend, done) { // use prior sig to construct new string to sign - const stringToSign = constructChunkStringToSign(this.timestamp, - this.credentialScope, this.lastSignature, dataToSend); - this.log.trace('constructed chunk string to sign', - { stringToSign }); + const stringToSign = constructChunkStringToSign( + this.timestamp, + this.credentialScope, + this.lastSignature, + dataToSend, + ); + this.log.trace('constructed chunk string to sign', { stringToSign }); // once used prior sig to construct string to sign, reassign // lastSignature to current signature this.lastSignature = this.currentSignature; @@ -167,15 +164,16 @@ class V4Transform extends Transform { }; return vault.authenticateV4Request(vaultParams, null, {}, err => { if (err) { - this.log.trace('err from vault on streaming v4 auth', - { error: err, paramsSentToVault: vaultParams.data }); + this.log.trace('err from vault on streaming v4 auth', { + error: err, + paramsSentToVault: vaultParams.data, + }); return done(err); } return done(); }); } - /** * This function will parse the chunk into metadata and data, * use the metadata to authenticate with vault and send the @@ -199,9 +197,7 @@ class V4Transform extends Transform { } if (this.lastPieceDone) { const slice = chunk.slice(0, 10); - this.log.trace('received chunk after end.' + - 'See first 10 bytes of chunk', - { chunk: slice.toString() }); + this.log.trace('received chunk after end.' + 'See first 10 bytes of chunk', { chunk: slice.toString() }); return callback(); } let unparsedChunk = chunk; @@ -212,11 +208,9 @@ class V4Transform extends Transform { // async function done => { if (!this.haveMetadata) { - this.log.trace('do not have metadata so calling ' + - '_parseMetadata'); + this.log.trace('do not have metadata so calling ' + '_parseMetadata'); // need to parse our metadata - const parsedMetadataResults = - this._parseMetadata(unparsedChunk); + const parsedMetadataResults = this._parseMetadata(unparsedChunk); if (parsedMetadataResults.err) { return done(parsedMetadataResults.err); } @@ -250,8 +244,7 @@ class V4Transform extends Transform { } // parse just the next data piece without \r\n at the end // (therefore, minus 2) - const nextDataPiece = - unparsedChunk.slice(0, this.seekingDataSize - 2); + const nextDataPiece = unparsedChunk.slice(0, this.seekingDataSize - 2); // add parsed data piece to other currentData pieces // so that this.currentData is the full data piece nextDataPiece.copy(this.currentData, this.dataCursor); @@ -259,8 +252,7 @@ class V4Transform extends Transform { if (err) { return done(err); } - unparsedChunk = - unparsedChunk.slice(this.seekingDataSize); + unparsedChunk = unparsedChunk.slice(this.seekingDataSize); this.push(this.currentData); this.haveMetadata = false; this.seekingDataSize = -1; @@ -283,7 +275,7 @@ class V4Transform extends Transform { } // get next chunk return callback(); - } + }, ); } } diff --git a/lib/auth/streamingV4/constructChunkStringToSign.js b/lib/auth/streamingV4/constructChunkStringToSign.js index 370501fe57..6a39806f93 100644 --- a/lib/auth/streamingV4/constructChunkStringToSign.js +++ b/lib/auth/streamingV4/constructChunkStringToSign.js @@ -13,20 +13,20 @@ const constants = require('../../../constants'); * @param {string} justDataChunk - data portion of chunk * @returns {string} stringToSign */ -function constructChunkStringToSign(timestamp, - credentialScope, lastSignature, justDataChunk) { +function constructChunkStringToSign(timestamp, credentialScope, lastSignature, justDataChunk) { let currentChunkHash; // for last chunk, there will be no data, so use emptyStringHash if (!justDataChunk) { currentChunkHash = constants.emptyStringHash; } else { currentChunkHash = crypto.createHash('sha256'); - currentChunkHash = currentChunkHash - .update(justDataChunk, 'binary').digest('hex'); + currentChunkHash = currentChunkHash.update(justDataChunk, 'binary').digest('hex'); } - return `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` + + return ( + `AWS4-HMAC-SHA256-PAYLOAD\n${timestamp}\n` + `${credentialScope}\n${lastSignature}\n` + - `${constants.emptyStringHash}\n${currentChunkHash}`; + `${constants.emptyStringHash}\n${currentChunkHash}` + ); } module.exports = constructChunkStringToSign; diff --git a/lib/auth/streamingV4/trailingChecksumTransform.js b/lib/auth/streamingV4/trailingChecksumTransform.js index 52712d4a4d..03e3586a79 100644 --- a/lib/auth/streamingV4/trailingChecksumTransform.js +++ b/lib/auth/streamingV4/trailingChecksumTransform.js @@ -3,7 +3,8 @@ const { errors, errorInstances } = require('arsenal'); const { maximumAllowedPartSize } = require('../../../constants'); const incompleteBodyError = errorInstances.IncompleteBody.customizeDescription( - 'The request body terminated unexpectedly'); + 'The request body terminated unexpectedly', +); const CR = '\r'.charCodeAt(0); const LF = '\n'.charCodeAt(0); @@ -231,8 +232,7 @@ class TrailingChecksumTransform extends Transform { // COMPLETED means we saw "0\r\n" but no trailer bytes after, // ChecksumTransform will enforce the trailer if x-amz-trailer was present. if (this.fsm.state !== State.COMPLETED_WITH_TRAILER && this.fsm.state !== State.COMPLETED_NO_TRAILER) { - this.log.error('stream ended without closing chunked encoding', - { state: this.fsm.state }); + this.log.error('stream ended without closing chunked encoding', { state: this.fsm.state }); return callback(incompleteBodyError); } diff --git a/lib/auth/vault.js b/lib/auth/vault.js index 3aeb06eaf9..5642e17563 100644 --- a/lib/auth/vault.js +++ b/lib/auth/vault.js @@ -49,20 +49,17 @@ function getMemBackend(config) { } switch (config.backends.auth) { -case 'mem': - implName = 'vaultMem'; - client = getMemBackend(config); - break; -case 'multiple': - implName = 'vaultChain'; - client = new ChainBackend('s3', [ - getMemBackend(config), - getVaultClient(config), - ]); - break; -default: // vault - implName = 'vault'; - client = getVaultClient(config); + case 'mem': + implName = 'vaultMem'; + client = getMemBackend(config); + break; + case 'multiple': + implName = 'vaultChain'; + client = new ChainBackend('s3', [getMemBackend(config), getVaultClient(config)]); + break; + default: // vault + implName = 'vault'; + client = getVaultClient(config); } module.exports = new Vault(client, implName); diff --git a/lib/data/wrapper.js b/lib/data/wrapper.js index cf3a216d4d..68caaf7f91 100644 --- a/lib/data/wrapper.js +++ b/lib/data/wrapper.js @@ -4,8 +4,7 @@ const { config } = require('../Config'); const kms = require('../kms/wrapper'); const metadata = require('../metadata/wrapper'); const vault = require('../auth/vault'); -const locationStorageCheck = - require('../api/apiUtils/object/locationStorageCheck'); +const locationStorageCheck = require('../api/apiUtils/object/locationStorageCheck'); const { DataWrapper, MultipleBackendGateway, parseLC } = storage.data; const { DataFileInterface } = storage.data.file; const inMemory = storage.data.inMemory.datastore.backend; @@ -28,8 +27,7 @@ if (config.backends.data === 'mem') { implName = 'file'; } else if (config.backends.data === 'multiple') { const clients = parseLC(config, vault); - client = new MultipleBackendGateway( - clients, metadata, locationStorageCheck); + client = new MultipleBackendGateway(clients, metadata, locationStorageCheck); implName = 'multipleBackends'; } else if (config.backends.data === 'cdmi') { if (!CdmiData) { @@ -45,14 +43,12 @@ if (config.backends.data === 'mem') { implName = 'cdmi'; } -const data = new DataWrapper( - client, implName, config, kms, metadata, locationStorageCheck, vault); +const data = new DataWrapper(client, implName, config, kms, metadata, locationStorageCheck, vault); config.on('location-constraints-update', () => { if (implName === 'multipleBackends') { const clients = parseLC(config, vault); - client = new MultipleBackendGateway( - clients, metadata, locationStorageCheck); + client = new MultipleBackendGateway(clients, metadata, locationStorageCheck); data.switch(client); } }); diff --git a/lib/kms/Cache.js b/lib/kms/Cache.js index c0ef71942a..43c8b8dbf2 100644 --- a/lib/kms/Cache.js +++ b/lib/kms/Cache.js @@ -41,7 +41,8 @@ class Cache { * @param {number} duration - Duration in milliseconds for cache validity. * @returns {boolean} true if the cache should be refreshed, else false. */ - shouldRefresh(duration = 1 * 60 * 60 * 1000) { // Default: 1 hour + shouldRefresh(duration = 1 * 60 * 60 * 1000) { + // Default: 1 hour if (!this.lastChecked) { return true; } @@ -49,7 +50,7 @@ class Cache { const now = Date.now(); const elapsed = now - this.lastChecked; const jitter = Math.floor(Math.random() * 15 * 60 * 1000); // Up to 15 minutes - return elapsed > (duration - jitter); + return elapsed > duration - jitter; } /** diff --git a/lib/kms/common.js b/lib/kms/common.js index 2c0cd3658d..50e87308a7 100644 --- a/lib/kms/common.js +++ b/lib/kms/common.js @@ -57,16 +57,16 @@ class Common { return newIV; } - /** - * Derive key to use in cipher - * @param {number} cryptoScheme - cryptoScheme being used - * @param {buffer} dataKey - the unencrypted key (either from the - * appliance on a get or originally generated by kms in the case of a put) - * @param {object} log - logger object - * @param {function} cb - cb from createDecipher - * @returns {undefined} - * @callback called with (err, derivedKey, derivedIV) - */ + /** + * Derive key to use in cipher + * @param {number} cryptoScheme - cryptoScheme being used + * @param {buffer} dataKey - the unencrypted key (either from the + * appliance on a get or originally generated by kms in the case of a put) + * @param {object} log - logger object + * @param {function} cb - cb from createDecipher + * @returns {undefined} + * @callback called with (err, derivedKey, derivedIV) + */ static _deriveKey(cryptoScheme, dataKey, log, cb) { if (cryptoScheme <= 1) { /* we are not storing hashed human password. @@ -79,69 +79,59 @@ class Common { */ const salt = Buffer.from('ItsTasty', 'utf8'); const iterations = 1; - return crypto.pbkdf2( - dataKey, salt, iterations, - this._keySize(), 'sha1', (err, derivedKey) => { + return crypto.pbkdf2(dataKey, salt, iterations, this._keySize(), 'sha1', (err, derivedKey) => { + if (err) { + log.error('pbkdf2 function failed on key derivation', { error: err }); + cb(errors.InternalError); + return; + } + crypto.pbkdf2(derivedKey, salt, iterations, this._IVSize(), 'sha1', (err, derivedIV) => { if (err) { - log.error('pbkdf2 function failed on key derivation', - { error: err }); - cb(errors.InternalError); - return; + log.error('pbkdf2 function failed on IV derivation', { error: err }); + return cb(errors.InternalError); } - crypto.pbkdf2( - derivedKey, salt, iterations, - this._IVSize(), 'sha1', (err, derivedIV) => { - if (err) { - log.error( - 'pbkdf2 function failed on IV derivation', - { error: err }); - return cb(errors.InternalError); - } - // derivedKey is the actual data encryption or - // decryption key used in the AES ctr cipher - return cb(null, derivedKey, derivedIV); - }); + // derivedKey is the actual data encryption or + // decryption key used in the AES ctr cipher + return cb(null, derivedKey, derivedIV); }); + }); } log.error('Unknown cryptographic scheme', { cryptoScheme }); return cb(errors.InternalError); } - /** - * createDecipher - * @param {number} cryptoScheme - cryptoScheme being used - * @param {buffer} dataKey - the unencrypted key (either from the - * appliance on a get or originally generated by kms in the case of a put) - * @param {number} offset - offset - * @param {object} log - logger object - * @param {function} cb - cb from external call - * @returns {undefined} - * @callback called with (err, decipher: ReadWritable.stream) - */ + /** + * createDecipher + * @param {number} cryptoScheme - cryptoScheme being used + * @param {buffer} dataKey - the unencrypted key (either from the + * appliance on a get or originally generated by kms in the case of a put) + * @param {number} offset - offset + * @param {object} log - logger object + * @param {function} cb - cb from external call + * @returns {undefined} + * @callback called with (err, decipher: ReadWritable.stream) + */ static createDecipher(cryptoScheme, dataKey, offset, log, cb) { - this._deriveKey( - cryptoScheme, dataKey, log, - (err, derivedKey, derivedIV) => { - if (err) { - log.debug('key derivation failed', { error: err }); - return cb(err); - } - const aesBlockSize = this._aesBlockSize(); - const blocks = Math.floor(offset / aesBlockSize); - const toSkip = offset % aesBlockSize; - const iv = this._incrementIV(derivedIV, blocks); - const cipher = crypto.createDecipheriv(this._algorithm(), - derivedKey, iv); - if (toSkip) { - /* Above, we advanced to the latest boundary not + this._deriveKey(cryptoScheme, dataKey, log, (err, derivedKey, derivedIV) => { + if (err) { + log.debug('key derivation failed', { error: err }); + return cb(err); + } + const aesBlockSize = this._aesBlockSize(); + const blocks = Math.floor(offset / aesBlockSize); + const toSkip = offset % aesBlockSize; + const iv = this._incrementIV(derivedIV, blocks); + const cipher = crypto.createDecipheriv(this._algorithm(), derivedKey, iv); + if (toSkip) { + /* Above, we advanced to the latest boundary not greater than the offset amount. Here we advance by the toSkip amount if necessary. */ - const dummyBuffer = Buffer.alloc(toSkip); - cipher.write(dummyBuffer); - cipher.read(); - } - return cb(null, cipher); - }); + const dummyBuffer = Buffer.alloc(toSkip); + cipher.write(dummyBuffer); + cipher.read(); + } + return cb(null, cipher); + }); } /** diff --git a/lib/kms/file/backend.js b/lib/kms/file/backend.js index 5f430b21ad..f7384f68b5 100644 --- a/lib/kms/file/backend.js +++ b/lib/kms/file/backend.js @@ -17,7 +17,7 @@ const backend = { * @param {function} cb - callback * @returns {undefined} * @callback called with (err, masterKeyId: string) - */ + */ createBucketKey: function createBucketKeyMem(bucket, log, cb) { process.nextTick(() => { // Using createDataKey here for purposes of createBucketKeyMem @@ -44,88 +44,69 @@ const backend = { }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - master key; for the file backend - * the master key is the actual bucket master key rather than the key to - * retrieve the actual key from a dictionary - * @param {buffer} plainTextDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, cipheredDataKey: Buffer) - */ - cipherDataKey: function cipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - plainTextDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - master key; for the file backend + * the master key is the actual bucket master key rather than the key to + * retrieve the actual key from a dictionary + * @param {buffer} plainTextDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, cipheredDataKey: Buffer) + */ + cipherDataKey: function cipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, plainTextDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); const masterKey = Buffer.from(masterKeyId, 'hex'); - Common.createCipher( - cryptoScheme, masterKey, 0, log, - (err, cipher) => { - if (err) { - cb(err); - return; - } - let cipheredDataKey = - cipher.update(plainTextDataKey); - // call final() to ensure that any bytes remaining in - // the output of the stream are captured - const final = cipher.final(); - if (final.length !== 0) { - cipheredDataKey = - Buffer.concat([cipheredDataKey, - final]); - } - cb(null, cipheredDataKey); - }); + Common.createCipher(cryptoScheme, masterKey, 0, log, (err, cipher) => { + if (err) { + cb(err); + return; + } + let cipheredDataKey = cipher.update(plainTextDataKey); + // call final() to ensure that any bytes remaining in + // the output of the stream are captured + const final = cipher.final(); + if (final.length !== 0) { + cipheredDataKey = Buffer.concat([cipheredDataKey, final]); + } + cb(null, cipheredDataKey); + }); }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - master key; for the file backend - * the master key is the actual bucket master key rather than the key to - * retrieve the actual key from a dictionary - * @param {buffer} cipheredDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, plainTextDataKey: Buffer) - */ - decipherDataKey: function decipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - cipheredDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - master key; for the file backend + * the master key is the actual bucket master key rather than the key to + * retrieve the actual key from a dictionary + * @param {buffer} cipheredDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, plainTextDataKey: Buffer) + */ + decipherDataKey: function decipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, cipheredDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); const masterKey = Buffer.from(masterKeyId, 'hex'); - Common.createDecipher( - cryptoScheme, masterKey, 0, log, - (err, decipher) => { - if (err) { - cb(err); - return; - } - let plainTextDataKey = - decipher.update(cipheredDataKey); - const final = decipher.final(); - if (final.length !== 0) { - plainTextDataKey = - Buffer.concat([plainTextDataKey, - final]); - } - cb(null, plainTextDataKey); - }); + Common.createDecipher(cryptoScheme, masterKey, 0, log, (err, decipher) => { + if (err) { + cb(err); + return; + } + let plainTextDataKey = decipher.update(cipheredDataKey); + const final = decipher.final(); + if (final.length !== 0) { + plainTextDataKey = Buffer.concat([plainTextDataKey, final]); + } + cb(null, plainTextDataKey); + }); }); }, - }; module.exports = backend; diff --git a/lib/kms/in_memory/backend.js b/lib/kms/in_memory/backend.js index 609ab17333..15e76e519c 100644 --- a/lib/kms/in_memory/backend.js +++ b/lib/kms/in_memory/backend.js @@ -15,14 +15,14 @@ const backend = { supportsDefaultKeyPerAccount: false, - /** - * - * @param {BucketInfo} bucket - bucket info - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, masterKeyId: string) - */ + /** + * + * @param {BucketInfo} bucket - bucket info + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, masterKeyId: string) + */ createBucketKey: function createBucketKeyMem(bucket, log, cb) { process.nextTick(() => { // Using createDataKey here for purposes of createBucketKeyMem @@ -49,82 +49,63 @@ const backend = { }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - key to retrieve master key - * @param {buffer} plainTextDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, cipheredDataKey: Buffer) - */ - cipherDataKey: function cipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - plainTextDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - key to retrieve master key + * @param {buffer} plainTextDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, cipheredDataKey: Buffer) + */ + cipherDataKey: function cipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, plainTextDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); - Common.createCipher( - cryptoScheme, kms[masterKeyId], 0, log, - (err, cipher) => { - if (err) { - cb(err); - return; - } - let cipheredDataKey = - cipher.update(plainTextDataKey); - // call final() to ensure that any bytes remaining in - // the output of the stream are captured - const final = cipher.final(); - if (final.length !== 0) { - cipheredDataKey = - Buffer.concat([cipheredDataKey, - final]); - } - cb(null, cipheredDataKey); - }); + Common.createCipher(cryptoScheme, kms[masterKeyId], 0, log, (err, cipher) => { + if (err) { + cb(err); + return; + } + let cipheredDataKey = cipher.update(plainTextDataKey); + // call final() to ensure that any bytes remaining in + // the output of the stream are captured + const final = cipher.final(); + if (final.length !== 0) { + cipheredDataKey = Buffer.concat([cipheredDataKey, final]); + } + cb(null, cipheredDataKey); + }); }); }, - /** - * - * @param {number} cryptoScheme - crypto scheme version number - * @param {string} masterKeyIdOrArn - key to retrieve master key - * @param {buffer} cipheredDataKey - data key - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, plainTextDataKey: Buffer) - */ - decipherDataKey: function decipherDataKeyMem(cryptoScheme, - masterKeyIdOrArn, - cipheredDataKey, - log, - cb) { + /** + * + * @param {number} cryptoScheme - crypto scheme version number + * @param {string} masterKeyIdOrArn - key to retrieve master key + * @param {buffer} cipheredDataKey - data key + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, plainTextDataKey: Buffer) + */ + decipherDataKey: function decipherDataKeyMem(cryptoScheme, masterKeyIdOrArn, cipheredDataKey, log, cb) { process.nextTick(() => { const masterKeyId = getKeyIdFromArn(masterKeyIdOrArn); - Common.createDecipher( - cryptoScheme, kms[masterKeyId], 0, log, - (err, decipher) => { - if (err) { - cb(err); - return; - } - let plainTextDataKey = - decipher.update(cipheredDataKey); - const final = decipher.final(); - if (final.length !== 0) { - plainTextDataKey = - Buffer.concat([plainTextDataKey, - final]); - } - cb(null, plainTextDataKey); - }); + Common.createDecipher(cryptoScheme, kms[masterKeyId], 0, log, (err, decipher) => { + if (err) { + cb(err); + return; + } + let plainTextDataKey = decipher.update(cipheredDataKey); + const final = decipher.final(); + if (final.length !== 0) { + plainTextDataKey = Buffer.concat([plainTextDataKey, final]); + } + cb(null, plainTextDataKey); + }); }); }, - }; module.exports = { diff --git a/lib/kms/utilities.js b/lib/kms/utilities.js index 5cdfa1f3bc..4c290528e0 100644 --- a/lib/kms/utilities.js +++ b/lib/kms/utilities.js @@ -5,13 +5,7 @@ const http = require('http'); const https = require('https'); const logger = require('../utilities/logger'); -function _createEncryptedBucket(host, - port, - bucketName, - accessKey, - secretKey, - verbose, ssl, - locationConstraint) { +function _createEncryptedBucket(host, port, bucketName, accessKey, secretKey, verbose, ssl, locationConstraint) { const options = { host, port, @@ -55,10 +49,11 @@ function _createEncryptedBucket(host, logger.info('request headers', { headers: request.getHeaders() }); } if (locationConstraint) { - const createBucketConfiguration = '' + - `${locationConstraint}` + - ''; + const createBucketConfiguration = + '' + + `${locationConstraint}` + + ''; request.write(createBucketConfiguration); } request.end(); @@ -81,20 +76,17 @@ function createEncryptedBucket() { .option('-p, --port ', 'Port of the server') .option('-s, --ssl', 'Enable ssl') .option('-v, --verbose') - .option('-l, --location-constraint ', - 'location Constraint') + .option('-l, --location-constraint ', 'location Constraint') .parse(process.argv); - const { host, port, accessKey, secretKey, bucket, verbose, ssl, - locationConstraint } = commander; + const { host, port, accessKey, secretKey, bucket, verbose, ssl, locationConstraint } = commander; if (!host || !port || !accessKey || !secretKey || !bucket) { logger.error('missing parameter'); commander.outputHelp(); process.exit(1); } - _createEncryptedBucket(host, port, bucket, accessKey, secretKey, verbose, - ssl, locationConstraint); + _createEncryptedBucket(host, port, bucket, accessKey, secretKey, verbose, ssl, locationConstraint); } module.exports = { diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index 772f6d2204..c2db3f2a06 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -30,9 +30,7 @@ function getScalityKms() { scalityKMS = new ScalityKMS(config.kms); scalityKMSImpl = 'scalityKms'; } catch (error) { - logger.warn('scality kms unavailable. ' + - 'Using file kms backend unless mem specified.', - { error }); + logger.warn('scality kms unavailable. ' + 'Using file kms backend unless mem specified.', { error }); scalityKMS = file; scalityKMSImpl = 'fileKms'; } @@ -97,12 +95,10 @@ if (config.sseMigration) { previousBackend = makeBackend( config.sseMigration.previousKeyType, config.sseMigration.previousKeyProtocol, - config.sseMigration.previousKeyProvider + config.sseMigration.previousKeyProvider, ); availableBackends.push(previousBackend); - previousIdentifier = `${previousBackend.type - }:${previousBackend.protocol - }:${previousBackend.provider}`; + previousIdentifier = `${previousBackend.type}:${previousBackend.protocol}:${previousBackend.provider}`; // Pre instantiate previous backend as for now only internal backend (file) is supported // for future multiple external backend we should consider keeping open connection to @@ -152,10 +148,10 @@ function getClientForKey(key, log) { // Only pre instantiated previous KMS from sseMigration is supported now // Here we could instantiate other provider on the fly to manage multi providers - log.error('KMS key doesn\'t match any KMS instance', { key, detail, availableBackends }); - return { error: new errors.InvalidArgument - // eslint-disable-next-line new-cap - .customizeDescription(`KMS unknown provider for key ${key}`), + log.error("KMS key doesn't match any KMS instance", { key, detail, availableBackends }); + return { + error: new // eslint-disable-next-line new-cap + errors.InvalidArgument.customizeDescription(`KMS unknown provider for key ${key}`), }; } @@ -170,20 +166,20 @@ class KMS { return client.backend.arnPrefix; } - /** - * Create a new bucket encryption key. - * - * This function is responsible for creating an encryption key for a bucket. - * If the client supports using a default master encryption key per account - * and one is configured, the key is managed at the account level by Vault. - * Otherwise, a bucket-level encryption key is created for legacy support. - * - * @param {BucketInfo} bucket - bucket info - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, { masterKeyId: string, masterKeyArn: string, isAccountEncryptionEnabled: boolean }) - */ + /** + * Create a new bucket encryption key. + * + * This function is responsible for creating an encryption key for a bucket. + * If the client supports using a default master encryption key per account + * and one is configured, the key is managed at the account level by Vault. + * Otherwise, a bucket-level encryption key is created for legacy support. + * + * @param {BucketInfo} bucket - bucket info + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, { masterKeyId: string, masterKeyArn: string, isAccountEncryptionEnabled: boolean }) + */ static createBucketKey(bucket, log, cb) { // always use current client for create log.debug('creating a new bucket key'); @@ -194,14 +190,19 @@ class KMS { if (client.supportsDefaultKeyPerAccount && config.defaultEncryptionKeyPerAccount) { return vault.getOrCreateEncryptionKeyId(bucket.getOwner(), log, (err, data) => { if (err) { - log.debug('error retrieving or creating the default encryption key at the account level from vault', - { implName, error: err }); + log.debug( + 'error retrieving or creating the default encryption key at the account level from vault', + { implName, error: err }, + ); return cb(err); } const { encryptionKeyId, action } = data; - log.trace('default encryption key retrieved or created at the account level from vault', - { implName, encryptionKeyId, action }); + log.trace('default encryption key retrieved or created at the account level from vault', { + implName, + encryptionKeyId, + action, + }); return cb(null, { // vault only return arn masterKeyId: encryptionKeyId, @@ -221,15 +222,15 @@ class KMS { }); } - /** - * - * @param {BucketInfo} bucket - bucket info - * @param {object} sseConfig - SSE configuration - * @param {object} log - logger object - * @param {function} cb - callback - * @returns {undefined} - * @callback called with (err, serverSideEncryptionInfo: object) - */ + /** + * + * @param {BucketInfo} bucket - bucket info + * @param {object} sseConfig - SSE configuration + * @param {object} log - logger object + * @param {function} cb - callback + * @returns {undefined} + * @callback called with (err, serverSideEncryptionInfo: object) + */ static bucketLevelEncryption(bucket, sseConfig, log, cb) { /* The purpose of bucket level encryption is so that the client does not @@ -258,8 +259,7 @@ class KMS { } serverSideEncryptionInfo.configuredMasterKeyId = configuredMasterKeyId; } else { - serverSideEncryptionInfo.configuredMasterKeyId = - `${client.backend.arnPrefix}${configuredMasterKeyId}`; + serverSideEncryptionInfo.configuredMasterKeyId = `${client.backend.arnPrefix}${configuredMasterKeyId}`; } return process.nextTick(() => cb(null, serverSideEncryptionInfo)); @@ -280,9 +280,9 @@ class KMS { return cb(null, serverSideEncryptionInfo); }); } - /* - * no encryption - */ + /* + * no encryption + */ return cb(null, null); } @@ -311,26 +311,25 @@ class KMS { }); } - /** - * createCipherBundle - * @param {object} serverSideEncryptionInfo - info for encryption - * @param {number} serverSideEncryptionInfo.cryptoScheme - - * cryptoScheme used - * @param {string} serverSideEncryptionInfo.algorithm - - * algorithm to use - * @param {string} serverSideEncryptionInfo.masterKeyId - - * key to get master key - * @param {boolean} serverSideEncryptionInfo.mandatory - - * true for mandatory encryption - * @param {object} log - logger object - * @param {function} cb - cb from external call - * @param {object} [opts] - additional options - * @param {boolean} [opts.previousOk] - allow usage of previous KMS (for ongoing MPU not migrated) - * @returns {undefined} - * @callback called with (err, cipherBundle) - */ - static createCipherBundle(serverSideEncryptionInfo, - log, cb, opts) { + /** + * createCipherBundle + * @param {object} serverSideEncryptionInfo - info for encryption + * @param {number} serverSideEncryptionInfo.cryptoScheme - + * cryptoScheme used + * @param {string} serverSideEncryptionInfo.algorithm - + * algorithm to use + * @param {string} serverSideEncryptionInfo.masterKeyId - + * key to get master key + * @param {boolean} serverSideEncryptionInfo.mandatory - + * true for mandatory encryption + * @param {object} log - logger object + * @param {function} cb - cb from external call + * @param {object} [opts] - additional options + * @param {boolean} [opts.previousOk] - allow usage of previous KMS (for ongoing MPU not migrated) + * @returns {undefined} + * @callback called with (err, cipherBundle) + */ + static createCipherBundle(serverSideEncryptionInfo, log, cb, opts) { const { algorithm, configuredMasterKeyId, masterKeyId: bucketMasterKeyId } = serverSideEncryptionInfo; let masterKeyId = bucketMasterKeyId; @@ -345,14 +344,18 @@ class KMS { if (error) { return cb(error); } - if (previousIdentifier - && clientIdentifier === previousIdentifier - && clientIdentifier !== currentIdentifier - && (opts && !opts.previousOk) + if ( + previousIdentifier && + clientIdentifier === previousIdentifier && + clientIdentifier !== currentIdentifier && + opts && + !opts.previousOk ) { - return cb(errors.InvalidArgument - .customizeDescription( - 'KMS cannot use previous provider to encrypt new objects if a new provider is configured')); + return cb( + errors.InvalidArgument.customizeDescription( + 'KMS cannot use previous provider to encrypt new objects if a new provider is configured', + ), + ); } const cipherBundle = { @@ -363,9 +366,10 @@ class KMS { cipher: null, }; - return async.waterfall([ - function generateDataKey(next) { - /* There are 2 ways of generating a datakey : + return async.waterfall( + [ + function generateDataKey(next) { + /* There are 2 ways of generating a datakey : - using the generateDataKey of the KMS backend if it exists (currently only implemented for the AWS KMS backend). This is the preferred solution since a dedicated KMS should offer a better @@ -374,92 +378,96 @@ class KMS { encrypt the datakey. This method is used when the KMS backend doesn't provide the generateDataKey method. */ - let res; - if (client.generateDataKey) { - log.debug('creating a data key using the KMS'); - res = client.generateDataKey(cipherBundle.cryptoScheme, - key, - log, (err, plainTextDataKey, cipheredDataKey) => { - if (err) { - log.debug('error generating a new data key from KMS', - { implName, error: err }); - return next(err); - } - log.trace('data key generated by the kms'); - return next(null, plainTextDataKey, cipheredDataKey); - }); - } else { - log.debug('creating a new data key'); - const plainTextDataKey = Common.createDataKey(); - - log.debug('ciphering the data key'); - res = client.cipherDataKey(cipherBundle.cryptoScheme, - key, - plainTextDataKey, log, (err, cipheredDataKey) => { - if (err) { - log.debug('error encrypting the data key using KMS', - { implName, error: err }); - return next(err); - } - log.trace('data key ciphered by the kms'); - return next(null, plainTextDataKey, cipheredDataKey); - }); - } - return res; - }, - function createCipher(plainTextDataKey, cipheredDataKey, next) { - log.debug('creating a cipher'); - cipherBundle.cipheredDataKey = - cipheredDataKey.toString('base64'); - return Common.createCipher(cipherBundle.cryptoScheme, - plainTextDataKey, 0, log, (err, cipher) => { + let res; + if (client.generateDataKey) { + log.debug('creating a data key using the KMS'); + res = client.generateDataKey( + cipherBundle.cryptoScheme, + key, + log, + (err, plainTextDataKey, cipheredDataKey) => { + if (err) { + log.debug('error generating a new data key from KMS', { implName, error: err }); + return next(err); + } + log.trace('data key generated by the kms'); + return next(null, plainTextDataKey, cipheredDataKey); + }, + ); + } else { + log.debug('creating a new data key'); + const plainTextDataKey = Common.createDataKey(); + + log.debug('ciphering the data key'); + res = client.cipherDataKey( + cipherBundle.cryptoScheme, + key, + plainTextDataKey, + log, + (err, cipheredDataKey) => { + if (err) { + log.debug('error encrypting the data key using KMS', { implName, error: err }); + return next(err); + } + log.trace('data key ciphered by the kms'); + return next(null, plainTextDataKey, cipheredDataKey); + }, + ); + } + return res; + }, + function createCipher(plainTextDataKey, cipheredDataKey, next) { + log.debug('creating a cipher'); + cipherBundle.cipheredDataKey = cipheredDataKey.toString('base64'); + return Common.createCipher(cipherBundle.cryptoScheme, plainTextDataKey, 0, log, (err, cipher) => { plainTextDataKey.fill(0); if (err) { - log.debug('error from kms', - { implName, error: err }); + log.debug('error from kms', { implName, error: err }); return next(err); } log.trace('cipher created by the kms'); return next(null, cipher); }); + }, + function finishCipherBundle(cipher, next) { + cipherBundle.cipher = cipher; + return next(null, cipherBundle); + }, + ], + (err, cipherBundle) => { + if (err) { + log.error('error processing cipher bundle', { implName, error: err }); + } + return cb(err, cipherBundle); }, - function finishCipherBundle(cipher, next) { - cipherBundle.cipher = cipher; - return next(null, cipherBundle); - }, - ], (err, cipherBundle) => { - if (err) { - log.error('error processing cipher bundle', - { implName, error: err }); - } - return cb(err, cipherBundle); - }); + ); } - /** - * createDecipherBundle - * @param {object} serverSideEncryptionInfo - info for decryption - * @param {number} serverSideEncryptionInfo.cryptoScheme - - * cryptoScheme used - * @param {string} serverSideEncryptionInfo.algorithm - - * algorithm to use - * @param {string} serverSideEncryptionInfo.masterKeyId - - * key to get master key - * @param {boolean} serverSideEncryptionInfo.mandatory - - * true for mandatory encryption - * @param {buffer} serverSideEncryptionInfo.cipheredDataKey - - * ciphered data key - * @param {number} offset - offset for decryption - * @param {object} log - logger object - * @param {function} cb - cb from external call - * @returns {undefined} - * @callback called with (err, decipherBundle) - */ - static createDecipherBundle(serverSideEncryptionInfo, offset, - log, cb) { - if (!serverSideEncryptionInfo.masterKeyId || + /** + * createDecipherBundle + * @param {object} serverSideEncryptionInfo - info for decryption + * @param {number} serverSideEncryptionInfo.cryptoScheme - + * cryptoScheme used + * @param {string} serverSideEncryptionInfo.algorithm - + * algorithm to use + * @param {string} serverSideEncryptionInfo.masterKeyId - + * key to get master key + * @param {boolean} serverSideEncryptionInfo.mandatory - + * true for mandatory encryption + * @param {buffer} serverSideEncryptionInfo.cipheredDataKey - + * ciphered data key + * @param {number} offset - offset for decryption + * @param {object} log - logger object + * @param {function} cb - cb from external call + * @returns {undefined} + * @callback called with (err, decipherBundle) + */ + static createDecipherBundle(serverSideEncryptionInfo, offset, log, cb) { + if ( + !serverSideEncryptionInfo.masterKeyId || !serverSideEncryptionInfo.cipheredDataKey || - !serverSideEncryptionInfo.cryptoScheme) { + !serverSideEncryptionInfo.cryptoScheme + ) { log.error('Invalid cryptographic information', { implName }); return cb(errors.InternalError); } @@ -469,55 +477,61 @@ class KMS { }; // shadowing global client for key - implName already used can't be shadowed here - const { error, client, implName: _impl, key } = getClientForKey( - serverSideEncryptionInfo.masterKeyId, log); + const { error, client, implName: _impl, key } = getClientForKey(serverSideEncryptionInfo.masterKeyId, log); if (error) { return cb(error); } - return async.waterfall([ - function decipherDataKey(next) { - return client.decipherDataKey( - decipherBundle.cryptoScheme, - key, - serverSideEncryptionInfo.cipheredDataKey, - log, (err, plainTextDataKey) => { - log.debug('deciphering a data key'); - if (err) { - log.debug('error from kms', - { implName: _impl, error: err }); - return next(err); - } - log.trace('data key deciphered by the kms'); - return next(null, plainTextDataKey); - }); - }, - function createDecipher(plainTextDataKey, next) { - log.debug('creating a decipher'); - return Common.createDecipher(decipherBundle.cryptoScheme, - plainTextDataKey, offset, log, (err, decipher) => { - plainTextDataKey.fill(0); - if (err) { - log.debug('error from kms', - { implName: _impl, error: err }); - return next(err); - } - log.trace('decipher created by the kms'); - return next(null, decipher); - }); - }, - function finishDecipherBundle(decipher, next) { - decipherBundle.decipher = decipher; - return next(null, decipherBundle); + return async.waterfall( + [ + function decipherDataKey(next) { + return client.decipherDataKey( + decipherBundle.cryptoScheme, + key, + serverSideEncryptionInfo.cipheredDataKey, + log, + (err, plainTextDataKey) => { + log.debug('deciphering a data key'); + if (err) { + log.debug('error from kms', { implName: _impl, error: err }); + return next(err); + } + log.trace('data key deciphered by the kms'); + return next(null, plainTextDataKey); + }, + ); + }, + function createDecipher(plainTextDataKey, next) { + log.debug('creating a decipher'); + return Common.createDecipher( + decipherBundle.cryptoScheme, + plainTextDataKey, + offset, + log, + (err, decipher) => { + plainTextDataKey.fill(0); + if (err) { + log.debug('error from kms', { implName: _impl, error: err }); + return next(err); + } + log.trace('decipher created by the kms'); + return next(null, decipher); + }, + ); + }, + function finishDecipherBundle(decipher, next) { + decipherBundle.decipher = decipher; + return next(null, decipherBundle); + }, + ], + (err, decipherBundle) => { + if (err) { + log.error('error processing decipher bundle', { implName: _impl, error: err }); + return cb(err); + } + return cb(err, decipherBundle); }, - ], (err, decipherBundle) => { - if (err) { - log.error('error processing decipher bundle', - { implName: _impl, error: err }); - return cb(err); - } - return cb(err, decipherBundle); - }); + ); } static checkHealth(log, cb) { diff --git a/lib/management/agentClient.js b/lib/management/agentClient.js index f88b199565..e10490766f 100644 --- a/lib/management/agentClient.js +++ b/lib/management/agentClient.js @@ -6,7 +6,6 @@ const _config = require('../Config').config; const { patchConfiguration } = require('./configuration'); const { reshapeExceptionError } = arsenal.errorUtils; - const managementAgentMessageType = { /** Message that contains the loaded overlay */ NEW_OVERLAY: 1, @@ -14,7 +13,6 @@ const managementAgentMessageType = { const CONNECTION_RETRY_TIMEOUT_MS = 5000; - function initManagementClient() { const { host, port } = _config.managementAgent; @@ -62,22 +60,22 @@ function initManagementClient() { } switch (msg.messageType) { - case managementAgentMessageType.NEW_OVERLAY: - patchConfiguration(msg.payload, log, err => { - if (err) { - log.error('failed to patch overlay', { - error: reshapeExceptionError(err), - method, - }); - } - }); - return; - default: - log.error('new overlay message with unmanaged message type', { - method, - type: msg.messageType, - }); - return; + case managementAgentMessageType.NEW_OVERLAY: + patchConfiguration(msg.payload, log, err => { + if (err) { + log.error('failed to patch overlay', { + error: reshapeExceptionError(err), + method, + }); + } + }); + return; + default: + log.error('new overlay message with unmanaged message type', { + method, + type: msg.messageType, + }); + return; } }); } @@ -86,7 +84,6 @@ function isManagementAgentUsed() { return process.env.MANAGEMENT_USE_AGENT === '1'; } - module.exports = { managementAgentMessageType, initManagementClient, diff --git a/lib/management/configuration.js b/lib/management/configuration.js index bd3cc08bea..de6db45995 100644 --- a/lib/management/configuration.js +++ b/lib/management/configuration.js @@ -19,9 +19,10 @@ function overlayHasVersion(overlay) { } function remoteOverlayIsNewer(cachedOverlay, remoteOverlay) { - return (overlayHasVersion(remoteOverlay) && - (!overlayHasVersion(cachedOverlay) || - remoteOverlay.version > cachedOverlay.version)); + return ( + overlayHasVersion(remoteOverlay) && + (!overlayHasVersion(cachedOverlay) || remoteOverlay.version > cachedOverlay.version) + ); } /** @@ -41,10 +42,8 @@ function patchConfiguration(newConf, log, cb) { return process.nextTick(cb, null, newConf); } - if (_config.overlayVersion !== undefined && - newConf.version <= _config.overlayVersion) { - log.debug('configuration version already applied', - { configurationVersion: newConf.version }); + if (_config.overlayVersion !== undefined && newConf.version <= _config.overlayVersion) { + log.debug('configuration version already applied', { configurationVersion: newConf.version }); return process.nextTick(cb, null, newConf); } return getStoredCredentials(log, (err, creds) => { @@ -62,8 +61,12 @@ function patchConfiguration(newConf, log, cb) { serviceName = u.accountType.split('-')[1]; } const newAccount = buildAuthDataAccount( - u.accessKey, secretKey, u.canonicalId, serviceName, - u.userName); + u.accessKey, + secretKey, + u.canonicalId, + serviceName, + u.userName, + ); accounts.push(newAccount.accounts[0]); } }); @@ -86,29 +89,30 @@ function patchConfiguration(newConf, log, cb) { _config.setLocationConstraints(locations); } catch (error) { const exceptionError = reshapeExceptionError(error); - log.error('could not apply configuration version location ' + - 'constraints', { error: exceptionError, - method: 'getStoredCredentials' }); + log.error('could not apply configuration version location ' + 'constraints', { + error: exceptionError, + method: 'getStoredCredentials', + }); return cb(exceptionError); } try { const locationsWithReplicationBackend = Object.keys(locations) - // NOTE: In Orbit, we don't need to have Scality location in our - // replication endpoind config, since we do not replicate to - // any Scality Instance yet. - .filter(key => replicationBackends - [locations[key].type]) - .reduce((obj, key) => { - /* eslint no-param-reassign:0 */ - obj[key] = locations[key]; - return obj; - }, {}); - _config.setReplicationEndpoints( - locationsWithReplicationBackend); + // NOTE: In Orbit, we don't need to have Scality location in our + // replication endpoind config, since we do not replicate to + // any Scality Instance yet. + .filter(key => replicationBackends[locations[key].type]) + .reduce((obj, key) => { + /* eslint no-param-reassign:0 */ + obj[key] = locations[key]; + return obj; + }, {}); + _config.setReplicationEndpoints(locationsWithReplicationBackend); } catch (error) { const exceptionError = reshapeExceptionError(error); - log.error('could not apply replication endpoints', - { error: exceptionError, method: 'getStoredCredentials' }); + log.error('could not apply replication endpoints', { + error: exceptionError, + method: 'getStoredCredentials', + }); return cb(exceptionError); } } @@ -118,18 +122,15 @@ function patchConfiguration(newConf, log, cb) { _config.setPublicInstanceId(newConf.instanceId); if (newConf.browserAccess) { - if (Boolean(_config.browserAccessEnabled) !== - Boolean(newConf.browserAccess.enabled)) { - _config.browserAccessEnabled = - Boolean(newConf.browserAccess.enabled); + if (Boolean(_config.browserAccessEnabled) !== Boolean(newConf.browserAccess.enabled)) { + _config.browserAccessEnabled = Boolean(newConf.browserAccess.enabled); _config.emit('browser-access-enabled-change'); } } _config.overlayVersion = newConf.version; - log.info('applied configuration version', - { configurationVersion: _config.overlayVersion }); + log.info('applied configuration version', { configurationVersion: _config.overlayVersion }); return cb(null, newConf); }); @@ -149,28 +150,33 @@ function patchConfiguration(newConf, log, cb) { function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) { if (remoteOverlayIsNewer(cachedOverlay, remoteOverlay)) { const objName = `configuration/overlay/${remoteOverlay.version}`; - metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay, - {}, log, error => { - if (error) { - const exceptionError = reshapeExceptionError(error); - log.error('could not save configuration', - { error: exceptionError, - method: 'saveConfigurationVersion', - configurationVersion: remoteOverlay.version }); - cb(exceptionError); - return; - } - metadata.putObjectMD(managementDatabaseName, - latestOverlayVersionKey, remoteOverlay.version, {}, log, - error => { - if (error) { - log.error('could not save configuration version', { - configurationVersion: remoteOverlay.version, - }); - } - cb(error, remoteOverlay); - }); - }); + metadata.putObjectMD(managementDatabaseName, objName, remoteOverlay, {}, log, error => { + if (error) { + const exceptionError = reshapeExceptionError(error); + log.error('could not save configuration', { + error: exceptionError, + method: 'saveConfigurationVersion', + configurationVersion: remoteOverlay.version, + }); + cb(exceptionError); + return; + } + metadata.putObjectMD( + managementDatabaseName, + latestOverlayVersionKey, + remoteOverlay.version, + {}, + log, + error => { + if (error) { + log.error('could not save configuration version', { + configurationVersion: remoteOverlay.version, + }); + } + cb(error, remoteOverlay); + }, + ); + }); } else { log.debug('no remote configuration to cache yet'); process.nextTick(cb, null, remoteOverlay); @@ -187,25 +193,29 @@ function saveConfigurationVersion(cachedOverlay, remoteOverlay, log, cb) { * @returns {undefined} */ function loadCachedOverlay(log, callback) { - return metadata.getObjectMD(managementDatabaseName, - latestOverlayVersionKey, {}, log, (err, version) => { - if (err) { - if (err.is.NoSuchKey) { - return process.nextTick(callback, null, {}); - } - return callback(err); + return metadata.getObjectMD(managementDatabaseName, latestOverlayVersionKey, {}, log, (err, version) => { + if (err) { + if (err.is.NoSuchKey) { + return process.nextTick(callback, null, {}); } - return metadata.getObjectMD(managementDatabaseName, - `configuration/overlay/${version}`, {}, log, (err, conf) => { - if (err) { - if (err.is.NoSuchKey) { - return process.nextTick(callback, null, {}); - } - return callback(err); + return callback(err); + } + return metadata.getObjectMD( + managementDatabaseName, + `configuration/overlay/${version}`, + {}, + log, + (err, conf) => { + if (err) { + if (err.is.NoSuchKey) { + return process.nextTick(callback, null, {}); } - return callback(null, conf); - }); - }); + return callback(err); + } + return callback(null, conf); + }, + ); + }); } function applyAndSaveOverlay(overlay, log) { diff --git a/lib/management/credentials.js b/lib/management/credentials.js index 814a731dc9..afee99d4f4 100644 --- a/lib/management/credentials.js +++ b/lib/management/credentials.js @@ -21,8 +21,7 @@ const { reshapeExceptionError } = arsenal.errorUtils; * @returns {undefined} */ function getStoredCredentials(log, callback) { - metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {}, - log, callback); + metadata.getObjectMD(managementDatabaseName, tokenConfigurationKey, {}, log, callback); } function issueCredentials(managementEndpoint, instanceId, log, callback) { @@ -36,8 +35,10 @@ function issueCredentials(managementEndpoint, instanceId, log, callback) { publicKey, }; - request.post(`${managementEndpoint}/${instanceId}/register`, - { body: postData, json: true }, (error, response, body) => { + request.post( + `${managementEndpoint}/${instanceId}/register`, + { body: postData, json: true }, + (error, response, body) => { if (error) { return callback(error); } @@ -51,11 +52,11 @@ function issueCredentials(managementEndpoint, instanceId, log, callback) { body.privateKey = privateKey; /* eslint-enable no-param-reassign */ return callback(null, body); - }); + }, + ); } -function confirmInstanceCredentials( - managementEndpoint, instanceId, creds, log, callback) { +function confirmInstanceCredentials(managementEndpoint, instanceId, creds, log, callback) { const postData = { serial: creds.serial || 0, publicKey: creds.publicKey, @@ -68,16 +69,15 @@ function confirmInstanceCredentials( body: postData, }; - request.post(`${managementEndpoint}/${instanceId}/confirm`, - opts, (error, response) => { - if (error) { - return callback(error); - } - if (response.statusCode === 200) { - return callback(null, instanceId, creds.token); - } - return callback(arsenal.errors.InternalError); - }); + request.post(`${managementEndpoint}/${instanceId}/confirm`, opts, (error, response) => { + if (error) { + return callback(error); + } + if (response.statusCode === 200) { + return callback(null, instanceId, creds.token); + } + return callback(arsenal.errors.InternalError); + }); } /** @@ -95,35 +95,37 @@ function confirmInstanceCredentials( * * @returns {undefined} */ -function initManagementCredentials( - managementEndpoint, instanceId, log, callback) { +function initManagementCredentials(managementEndpoint, instanceId, log, callback) { getStoredCredentials(log, (error, value) => { if (error) { if (error.is.NoSuchKey) { - return issueCredentials(managementEndpoint, instanceId, log, - (error, value) => { + return issueCredentials(managementEndpoint, instanceId, log, (error, value) => { if (error) { - log.error('could not issue token', - { error: reshapeExceptionError(error), - method: 'initManagementCredentials' }); + log.error('could not issue token', { + error: reshapeExceptionError(error), + method: 'initManagementCredentials', + }); return callback(error); } log.debug('saving token'); - return metadata.putObjectMD(managementDatabaseName, - tokenConfigurationKey, value, {}, log, error => { + return metadata.putObjectMD( + managementDatabaseName, + tokenConfigurationKey, + value, + {}, + log, + error => { if (error) { - log.error('could not save token', - { error: reshapeExceptionError(error), - method: 'initManagementCredentials', - }); + log.error('could not save token', { + error: reshapeExceptionError(error), + method: 'initManagementCredentials', + }); return callback(error); } - log.info('saved token locally, ' + - 'confirming instance'); - return confirmInstanceCredentials( - managementEndpoint, instanceId, value, log, - callback); - }); + log.info('saved token locally, ' + 'confirming instance'); + return confirmInstanceCredentials(managementEndpoint, instanceId, value, log, callback); + }, + ); }); } log.debug('could not get token', { error }); diff --git a/lib/management/index.js b/lib/management/index.js index d98f3c46bc..2b8b7d6755 100644 --- a/lib/management/index.js +++ b/lib/management/index.js @@ -4,11 +4,7 @@ const async = require('async'); const metadata = require('../metadata/wrapper'); const logger = require('../utilities/logger'); -const { - loadCachedOverlay, - managementDatabaseName, - patchConfiguration, -} = require('./configuration'); +const { loadCachedOverlay, managementDatabaseName, patchConfiguration } = require('./configuration'); const { initManagementCredentials } = require('./credentials'); const { startWSManagementClient } = require('./push'); const { startPollingManagementClient } = require('./poll'); @@ -17,20 +13,20 @@ const { isManagementAgentUsed } = require('./agentClient'); const initRemoteManagementRetryDelay = 10000; -const managementEndpointRoot = - process.env.MANAGEMENT_ENDPOINT || - 'https://api.zenko.io'; +const managementEndpointRoot = process.env.MANAGEMENT_ENDPOINT || 'https://api.zenko.io'; const managementEndpoint = `${managementEndpointRoot}/api/v1/instance`; -const pushEndpointRoot = - process.env.PUSH_ENDPOINT || - 'https://push.api.zenko.io'; +const pushEndpointRoot = process.env.PUSH_ENDPOINT || 'https://push.api.zenko.io'; const pushEndpoint = `${pushEndpointRoot}/api/v1/instance`; function initManagementDatabase(log, callback) { // XXX choose proper owner names - const md = new arsenal.models.BucketInfo(managementDatabaseName, 'owner', - 'owner display name', new Date().toJSON()); + const md = new arsenal.models.BucketInfo( + managementDatabaseName, + 'owner', + 'owner display name', + new Date().toJSON(), + ); metadata.createBucket(managementDatabaseName, md, log, error => { if (error) { @@ -38,9 +34,10 @@ function initManagementDatabase(log, callback) { log.info('created management database'); return callback(); } - log.error('could not initialize management database', - { error: reshapeExceptionError(error), - method: 'initManagementDatabase' }); + log.error('could not initialize management database', { + error: reshapeExceptionError(error), + method: 'initManagementDatabase', + }); return callback(error); } log.info('initialized management database'); @@ -75,61 +72,63 @@ function startManagementListeners(instanceId, token) { * @returns {undefined} */ function initManagement(log, callback) { - if ((process.env.REMOTE_MANAGEMENT_DISABLE && - process.env.REMOTE_MANAGEMENT_DISABLE !== '0') - || process.env.S3BACKEND === 'mem') { + if ( + (process.env.REMOTE_MANAGEMENT_DISABLE && process.env.REMOTE_MANAGEMENT_DISABLE !== '0') || + process.env.S3BACKEND === 'mem' + ) { log.info('remote management disabled'); return; } /* Temporary check before to fully move to the process management agent. */ - if (isManagementAgentUsed() ^ typeof callback === 'function') { + if (isManagementAgentUsed() ^ (typeof callback === 'function')) { let msg = 'misuse of initManagement function: '; msg += `MANAGEMENT_USE_AGENT: ${process.env.MANAGEMENT_USE_AGENT}`; msg += `, callback type: ${typeof callback}`; throw new Error(msg); } - async.waterfall([ - // eslint-disable-next-line arrow-body-style - cb => { return isManagementAgentUsed() ? metadata.setup(cb) : cb(); }, - cb => initManagementDatabase(log, cb), - cb => metadata.getUUID(log, cb), - (instanceId, cb) => initManagementCredentials( - managementEndpoint, instanceId, log, cb), - (instanceId, token, cb) => { - if (!isManagementAgentUsed()) { - cb(null, instanceId, token, {}); - return; + async.waterfall( + [ + // eslint-disable-next-line arrow-body-style + cb => { + return isManagementAgentUsed() ? metadata.setup(cb) : cb(); + }, + cb => initManagementDatabase(log, cb), + cb => metadata.getUUID(log, cb), + (instanceId, cb) => initManagementCredentials(managementEndpoint, instanceId, log, cb), + (instanceId, token, cb) => { + if (!isManagementAgentUsed()) { + cb(null, instanceId, token, {}); + return; + } + loadCachedOverlay(log, (err, overlay) => cb(err, instanceId, token, overlay)); + }, + (instanceId, token, overlay, cb) => { + if (!isManagementAgentUsed()) { + cb(null, instanceId, token, overlay); + return; + } + patchConfiguration(overlay, log, err => cb(err, instanceId, token, overlay)); + }, + ], + (error, instanceId, token, overlay) => { + if (error) { + log.error('could not initialize remote management, retrying later', { + error: reshapeExceptionError(error), + method: 'initManagement', + }); + setTimeout(initManagement, initRemoteManagementRetryDelay, logger.newRequestLogger()); + } else { + log.info(`this deployment's Instance ID is ${instanceId}`); + log.end('management init done'); + startManagementListeners(instanceId, token); + if (callback) { + callback(overlay); + } } - loadCachedOverlay(log, (err, overlay) => cb(err, instanceId, - token, overlay)); }, - (instanceId, token, overlay, cb) => { - if (!isManagementAgentUsed()) { - cb(null, instanceId, token, overlay); - return; - } - patchConfiguration(overlay, log, - err => cb(err, instanceId, token, overlay)); - }, - ], (error, instanceId, token, overlay) => { - if (error) { - log.error('could not initialize remote management, retrying later', - { error: reshapeExceptionError(error), - method: 'initManagement' }); - setTimeout(initManagement, - initRemoteManagementRetryDelay, - logger.newRequestLogger()); - } else { - log.info(`this deployment's Instance ID is ${instanceId}`); - log.end('management init done'); - startManagementListeners(instanceId, token); - if (callback) { - callback(overlay); - } - } - }); + ); } module.exports = { diff --git a/lib/management/push.js b/lib/management/push.js index 82ad9be7b5..01123ec376 100644 --- a/lib/management/push.js +++ b/lib/management/push.js @@ -14,25 +14,15 @@ const metadata = require('../metadata/wrapper'); const { reshapeExceptionError } = arsenal.errorUtils; const { isManagementAgentUsed } = require('./agentClient'); const { applyAndSaveOverlay } = require('./configuration'); -const { - ChannelMessageV0, - MessageType, -} = require('./ChannelMessageV0'); - -const { - CONFIG_OVERLAY_MESSAGE, - METRICS_REQUEST_MESSAGE, - CHANNEL_CLOSE_MESSAGE, - CHANNEL_PAYLOAD_MESSAGE, -} = MessageType; +const { ChannelMessageV0, MessageType } = require('./ChannelMessageV0'); + +const { CONFIG_OVERLAY_MESSAGE, METRICS_REQUEST_MESSAGE, CHANNEL_CLOSE_MESSAGE, CHANNEL_PAYLOAD_MESSAGE } = MessageType; const PING_INTERVAL_MS = 10000; const subprotocols = [ChannelMessageV0.protocolName]; -const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST - || 'localhost'; -const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT - || _config.port; +const cloudServerHost = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_HOST || 'localhost'; +const cloudServerPort = process.env.SECURE_CHANNEL_DEFAULT_FORWARD_TO_PORT || _config.port; let overlayMessageListener = null; let connected = false; @@ -40,8 +30,7 @@ let connected = false; // No wildcard nor cidr/mask match for now function createWSAgent(pushEndpoint, env, log) { const url = new _URL(pushEndpoint); - const noProxy = (env.NO_PROXY || env.no_proxy - || '').split(','); + const noProxy = (env.NO_PROXY || env.no_proxy || '').split(','); if (noProxy.includes(url.hostname)) { log.info('push server ws has proxy exclusion', { noProxy }); @@ -49,20 +38,20 @@ function createWSAgent(pushEndpoint, env, log) { } if (url.protocol === 'https:' || url.protocol === 'wss:') { - const httpsProxy = (env.HTTPS_PROXY || env.https_proxy); + const httpsProxy = env.HTTPS_PROXY || env.https_proxy; if (httpsProxy) { log.info('push server ws using https proxy', { httpsProxy }); return new HttpsProxyAgent(httpsProxy); } } else if (url.protocol === 'http:' || url.protocol === 'ws:') { - const httpProxy = (env.HTTP_PROXY || env.http_proxy); + const httpProxy = env.HTTP_PROXY || env.http_proxy; if (httpProxy) { log.info('push server ws using http proxy', { httpProxy }); return new HttpsProxyAgent(httpProxy); } } - const allProxy = (env.ALL_PROXY || env.all_proxy); + const allProxy = env.ALL_PROXY || env.all_proxy; if (allProxy) { log.info('push server ws using wildcard proxy', { allProxy }); return new HttpsProxyAgent(allProxy); @@ -88,8 +77,7 @@ function startWSManagementClient(url, token, cb) { logger.info('connecting to push server', { url }); function _logError(error, errorMessage, method) { if (error) { - logger.error(`management client error: ${errorMessage}`, - { error: reshapeExceptionError(error), method }); + logger.error(`management client error: ${errorMessage}`, { error: reshapeExceptionError(error), method }); } } @@ -131,9 +119,9 @@ function startWSManagementClient(url, token, cb) { _logError(err, 'failed to get metrics report', 'pushStats'); return; } - ws.send(ChannelMessageV0.encodeMetricsReportMessage(body), - err => _logError(err, 'failed to send metrics report message', - 'pushStats')); + ws.send(ChannelMessageV0.encodeMetricsReportMessage(body), err => + _logError(err, 'failed to send metrics report message', 'pushStats'), + ); }); } @@ -151,17 +139,14 @@ function startWSManagementClient(url, token, cb) { socket = net.createConnection(cloudServerPort, cloudServerHost); socket.on('data', data => { - ws.send(ChannelMessageV0. - encodeChannelDataMessage(channelId, data), err => - _logError(err, 'failed to send channel data message', - 'receiveChannelData')); + ws.send(ChannelMessageV0.encodeChannelDataMessage(channelId, data), err => + _logError(err, 'failed to send channel data message', 'receiveChannelData'), + ); }); - socket.on('connect', () => { - }); + socket.on('connect', () => {}); - socket.on('drain', () => { - }); + socket.on('drain', () => {}); socket.on('error', error => { logger.error('failed to connect to S3', { @@ -174,10 +159,9 @@ function startWSManagementClient(url, token, cb) { socket.on('end', () => { socket.destroy(); socketsByChannelId[channelId] = null; - ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId), - err => _logError(err, - 'failed to send channel close message', - 'receiveChannelData')); + ws.send(ChannelMessageV0.encodeChannelCloseMessage(channelId), err => + _logError(err, 'failed to send channel close message', 'receiveChannelData'), + ); }); socketsByChannelId[channelId] = socket; @@ -208,8 +192,7 @@ function startWSManagementClient(url, token, cb) { ws.on('close', () => { logger.info('disconnected from push server, reconnecting in 10s'); metadata.notifyBucketChange(null); - _config.removeListener('browser-access-enabled-change', - browserAccessChangeHandler); + _config.removeListener('browser-access-enabled-change', browserAccessChangeHandler); setTimeout(startWSManagementClient, 10000, url, token); connected = false; @@ -241,31 +224,29 @@ function startWSManagementClient(url, token, cb) { const log = logger.newRequestLogger(); const message = new ChannelMessageV0(data); switch (message.getType()) { - case CONFIG_OVERLAY_MESSAGE: - if (!isManagementAgentUsed()) { - applyAndSaveOverlay(JSON.parse(message.getPayload()), log); - } else { - if (overlayMessageListener) { - overlayMessageListener(message.getPayload().toString()); + case CONFIG_OVERLAY_MESSAGE: + if (!isManagementAgentUsed()) { + applyAndSaveOverlay(JSON.parse(message.getPayload()), log); + } else { + if (overlayMessageListener) { + overlayMessageListener(message.getPayload().toString()); + } } - } - break; - case METRICS_REQUEST_MESSAGE: - pushStats(); - break; - case CHANNEL_CLOSE_MESSAGE: - closeChannel(message.getChannelNumber()); - break; - case CHANNEL_PAYLOAD_MESSAGE: - // browserAccessEnabled defaults to true unless explicitly false - if (_config.browserAccessEnabled !== false) { - receiveChannelData( - message.getChannelNumber(), message.getPayload()); - } - break; - default: - logger.error('unknown message type from push server', - { messageType: message.getType() }); + break; + case METRICS_REQUEST_MESSAGE: + pushStats(); + break; + case CHANNEL_CLOSE_MESSAGE: + closeChannel(message.getChannelNumber()); + break; + case CHANNEL_PAYLOAD_MESSAGE: + // browserAccessEnabled defaults to true unless explicitly false + if (_config.browserAccessEnabled !== false) { + receiveChannelData(message.getChannelNumber(), message.getPayload()); + } + break; + default: + logger.error('unknown message type from push server', { messageType: message.getType() }); } }); } diff --git a/lib/metadata/wrapper.js b/lib/metadata/wrapper.js index 6bd800e60f..947d103847 100644 --- a/lib/metadata/wrapper.js +++ b/lib/metadata/wrapper.js @@ -39,6 +39,5 @@ if (clientName === 'mem') { }; } -const metadata = new MetadataWrapper(config.backends.metadata, params, - bucketclient, logger); +const metadata = new MetadataWrapper(config.backends.metadata, params, bucketclient, logger); module.exports = metadata; diff --git a/lib/nfs/utilities.js b/lib/nfs/utilities.js index d8d0a5e9f3..83a80146db 100644 --- a/lib/nfs/utilities.js +++ b/lib/nfs/utilities.js @@ -4,8 +4,7 @@ const http = require('http'); const https = require('https'); const logger = require('../utilities/logger'); -function _createBucketWithNFSEnabled(host, port, bucketName, accessKey, - secretKey, verbose, ssl, locationConstraint) { +function _createBucketWithNFSEnabled(host, port, bucketName, accessKey, secretKey, verbose, ssl, locationConstraint) { const options = { host, port, @@ -47,10 +46,11 @@ function _createBucketWithNFSEnabled(host, port, bucketName, accessKey, logger.info('request headers', { headers: request.getHeaders() }); } if (locationConstraint) { - const createBucketConfiguration = '' + - `${locationConstraint}` + - ''; + const createBucketConfiguration = + '' + + `${locationConstraint}` + + ''; request.write(createBucketConfiguration); } request.end(); @@ -73,20 +73,17 @@ function createBucketWithNFSEnabled() { .option('-p, --port ', 'Port of the server') .option('-s', '--ssl', 'Enable ssl') .option('-v, --verbose') - .option('-l, --location-constraint ', - 'location Constraint') + .option('-l, --location-constraint ', 'location Constraint') .parse(process.argv); - const { host, port, accessKey, secretKey, bucket, verbose, - ssl, locationConstraint } = commander; + const { host, port, accessKey, secretKey, bucket, verbose, ssl, locationConstraint } = commander; if (!host || !port || !accessKey || !secretKey || !bucket) { logger.error('missing parameter'); commander.outputHelp(); process.exit(1); } - _createBucketWithNFSEnabled(host, port, bucket, accessKey, secretKey, - verbose, ssl, locationConstraint); + _createBucketWithNFSEnabled(host, port, bucket, accessKey, secretKey, verbose, ssl, locationConstraint); } module.exports = { diff --git a/lib/routes/routeMetadata.js b/lib/routes/routeMetadata.js index a5cac213e2..64319b3c6e 100644 --- a/lib/routes/routeMetadata.js +++ b/lib/routes/routeMetadata.js @@ -45,68 +45,85 @@ function routeMetadata(clientIP, request, response, log) { // restrict access to only routes ending in bucket, log or id const { resourceType, subResource } = request; - if (resourceType === 'admin' - && !['bucket', 'log', 'id'].includes(subResource)) { + if (resourceType === 'admin' && !['bucket', 'log', 'id'].includes(subResource)) { return responseJSONBody(errors.NotImplemented, null, response, log); } const ip = requestUtils.getClientIp(request, config); const isSecure = requestUtils.getHttpProtocolSecurity(request, config); - const requestContexts = [new RequestContext(request.headers, request.query, - request.generalResource, request.specificResource, ip, - isSecure, request.resourceType, 'metadata')]; - return waterfall([ - next => auth.server.doAuth(request, log, (err, userInfo, authRes) => { - if (err) { - log.debug('authentication error', { - error: err, - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, - }); - return next(err); - } - // authRes is not defined for account credentials - if (authRes && !authRes[0].isAllowed) { - return next(errors.AccessDenied); - } - return next(null, userInfo); - }, 's3', requestContexts), - (userInfo, next) => { - if (userInfo.getCanonicalID() === constants.publicId) { - log.debug('unauthenticated access to API routes', { - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, - }); - return next(errors.AccessDenied); - } - const { url } = request; - const path = url.startsWith('/_/metadata/admin') ? - url.replace('/_/metadata/admin/', '/_/') : - url.replace('/_/metadata/', '/'); - // bucketd is always configured on the loopback interface in s3c - const endpoint = bootstrap[0]; - const target = `http://${endpoint}${path}`; - return metadataProxy.web(request, response, { target }, err => { - if (err) { - log.error('error proxying request to metadata admin server', - { error: err.message }); - return next(errors.ServiceUnavailable); + const requestContexts = [ + new RequestContext( + request.headers, + request.query, + request.generalResource, + request.specificResource, + ip, + isSecure, + request.resourceType, + 'metadata', + ), + ]; + return waterfall( + [ + next => + auth.server.doAuth( + request, + log, + (err, userInfo, authRes) => { + if (err) { + log.debug('authentication error', { + error: err, + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return next(err); + } + // authRes is not defined for account credentials + if (authRes && !authRes[0].isAllowed) { + return next(errors.AccessDenied); + } + return next(null, userInfo); + }, + 's3', + requestContexts, + ), + (userInfo, next) => { + if (userInfo.getCanonicalID() === constants.publicId) { + log.debug('unauthenticated access to API routes', { + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return next(errors.AccessDenied); } - return next(); - }); - }], + const { url } = request; + const path = url.startsWith('/_/metadata/admin') + ? url.replace('/_/metadata/admin/', '/_/') + : url.replace('/_/metadata/', '/'); + // bucketd is always configured on the loopback interface in s3c + const endpoint = bootstrap[0]; + const target = `http://${endpoint}${path}`; + return metadataProxy.web(request, response, { target }, err => { + if (err) { + log.error('error proxying request to metadata admin server', { error: err.message }); + return next(errors.ServiceUnavailable); + } + return next(); + }); + }, + ], err => { if (err) { return responseJSONBody(err, null, response, log); } - log.debug('metadata route response sent successfully', - { method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey }); + log.debug('metadata route response sent successfully', { + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); return undefined; - }); + }, + ); } - module.exports = routeMetadata; diff --git a/lib/routes/routeVeeam.js b/lib/routes/routeVeeam.js index bfd83ae1e5..b6dc8f4f80 100644 --- a/lib/routes/routeVeeam.js +++ b/lib/routes/routeVeeam.js @@ -16,10 +16,7 @@ const { responseXMLBody } = s3routes.routesUtils; auth.setHandler(vault); -const validObjectKeys = [ - `${validPath}system.xml`, - `${validPath}capacity.xml`, -]; +const validObjectKeys = [`${validPath}system.xml`, `${validPath}capacity.xml`]; const apiToAction = { PUT: 'PutObject', @@ -29,10 +26,7 @@ const apiToAction = { LIST: 'ListObjects', }; -const allowedSdkQueryKeys = new Set([ - 'x-id', - 'x-amz-user-agent', -]); +const allowedSdkQueryKeys = new Set(['x-id', 'x-amz-user-agent']); // Allowed query parameters for SigV4 presigned URLs (lower-cased). const allowedPresignQueryKeys = new Set([ @@ -91,17 +85,18 @@ function checkBucketAndKey(bucketName, objectKey, requestQueryParams, method, lo // Ensure x-id, when present, matches the expected action for the method. if (normalizedKey === 'x-id' && value !== apiToAction[method]) { - return errorInstances.InvalidRequest - .customizeDescription('The Veeam SOSAPI folder does not support this action.'); + return errorInstances.InvalidRequest.customizeDescription( + 'The Veeam SOSAPI folder does not support this action.', + ); } - const isAllowedSdkKey = allowedSdkQueryKeys.has(normalizedKey) - || normalizedKey.startsWith('x-amz-sdk-'); + const isAllowedSdkKey = allowedSdkQueryKeys.has(normalizedKey) || normalizedKey.startsWith('x-amz-sdk-'); const isAllowedPresignKey = allowedPresignQueryKeys.has(normalizedKey); if (!isAllowedSdkKey && !isAllowedPresignKey) { - return errorInstances.InvalidRequest - .customizeDescription('The Veeam SOSAPI folder does not support this action.'); + return errorInstances.InvalidRequest.customizeDescription( + 'The Veeam SOSAPI folder does not support this action.', + ); } } if (typeof objectKey !== 'string' || !validObjectKeys.includes(objectKey)) { @@ -128,44 +123,54 @@ function authorizationMiddleware(request, response, api, log, callback) { return responseXMLBody(errors.AccessDenied, null, response, log); } const requestContexts = prepareRequestContexts(api, request); - return async.waterfall([ - next => auth.server.doAuth(request, log, (err, userInfo, authorizationResults, streamingV4Params) => { - if (err) { - log.debug('authentication error', { - error: err, - method: request.method, + return async.waterfall( + [ + next => + auth.server.doAuth( + request, + log, + (err, userInfo, authorizationResults, streamingV4Params) => { + if (err) { + log.debug('authentication error', { + error: err, + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + } + /* eslint-disable no-param-reassign */ + request.authorizationResults = authorizationResults; + request.streamingV4Params = streamingV4Params; + /* eslint-enable no-param-reassign */ + return next(err, userInfo); + }, + 's3', + requestContexts, + ), + (userInfo, next) => { + // Ensure only supported HTTP verbs and actions are called, + // otherwise deny access + const requestType = apiToAction[api]; + if (!requestType) { + return next(errors.AccessDenied); + } + const mdValParams = { bucketName: request.bucketName, - objectKey: request.objectKey, - }); - } - /* eslint-disable no-param-reassign */ - request.authorizationResults = authorizationResults; - request.streamingV4Params = streamingV4Params; - /* eslint-enable no-param-reassign */ - return next(err, userInfo); - }, 's3', requestContexts), - (userInfo, next) => { - // Ensure only supported HTTP verbs and actions are called, - // otherwise deny access - const requestType = apiToAction[api]; - if (!requestType) { - return next(errors.AccessDenied); + authInfo: userInfo, + requestType, + request, + }; + return next(null, mdValParams); + }, + (mdValParams, next) => standardMetadataValidateBucket(mdValParams, request.actionImplicitDenies, log, next), + ], + (err, bucketMd) => { + if (err || !bucketMd) { + return responseXMLBody(err, null, response, log); } - const mdValParams = { - bucketName: request.bucketName, - authInfo: userInfo, - requestType, - request, - }; - return next(null, mdValParams); + return callback(request, response, bucketMd, log); }, - (mdValParams, next) => standardMetadataValidateBucket(mdValParams, request.actionImplicitDenies, log, next), - ], (err, bucketMd) => { - if (err || !bucketMd) { - return responseXMLBody(err, null, response, log); - } - return callback(request, response, bucketMd, log); - }); + ); } function _normalizeVeeamRequest(req) { @@ -183,11 +188,10 @@ function _normalizeVeeamRequest(req) { req.query = parsedUrl.query; req.bucketName = pathArr[1]; req.objectKey = pathArr.slice(2).join('/'); - const contentLength = req.headers['x-amz-decoded-content-length'] ? - req.headers['x-amz-decoded-content-length'] : - req.headers['content-length']; - req.parsedContentLength = - Number.parseInt(contentLength?.toString() ?? '', 10); + const contentLength = req.headers['x-amz-decoded-content-length'] + ? req.headers['x-amz-decoded-content-length'] + : req.headers['content-length']; + req.parsedContentLength = Number.parseInt(contentLength?.toString() ?? '', 10); /* eslint-enable no-param-reassign */ } @@ -237,11 +241,15 @@ function routeVeeam(clientIP, request, response, log) { return responseXMLBody(error, '', response, log); } const bucketOrKeyError = checkBucketAndKey( - request.bucketName, request.objectKey, request.query, requestMethod, log); + request.bucketName, + request.objectKey, + request.query, + requestMethod, + log, + ); if (bucketOrKeyError) { - log.error('error with bucket or key value', - { error: bucketOrKeyError }); + log.error('error with bucket or key value', { error: bucketOrKeyError }); return routesUtils.responseXMLBody(bucketOrKeyError, null, response, log); } return authorizationMiddleware(request, response, requestMethod, log, method); diff --git a/lib/routes/routeWorkflowEngineOperator.js b/lib/routes/routeWorkflowEngineOperator.js index bf73a1ed68..c231cf9e47 100644 --- a/lib/routes/routeWorkflowEngineOperator.js +++ b/lib/routes/routeWorkflowEngineOperator.js @@ -4,12 +4,10 @@ const httpProxy = require('http-proxy'); const workflowEngineOperatorProxy = httpProxy.createProxyServer({ ignorePath: true, }); -const { auth, errors, s3routes } = - require('arsenal'); +const { auth, errors, s3routes } = require('arsenal'); const { responseJSONBody } = s3routes.routesUtils; const vault = require('../auth/vault'); -const prepareRequestContexts = require( -'../api/apiUtils/authorization/prepareRequestContexts'); +const prepareRequestContexts = require('../api/apiUtils/authorization/prepareRequestContexts'); const { config } = require('../Config'); const constants = require('../../constants'); @@ -52,45 +50,45 @@ function routeWorkflowEngineOperator(clientIP, request, response, log) { log.debug('unable to proxy workflow engine operator request', { workflowEngineConfig: config.workflowEngineOperator, }); - return responseJSONBody(errors.MethodNotAllowed, null, response, - log); + return responseJSONBody(errors.MethodNotAllowed, null, response, log); } const path = request.url.replace('/_/workflow-engine-operator/api', '/_/'); const { host, port } = config.workflowEngineOperator; const target = `http://${host}:${port}${path}`; - return auth.server.doAuth(request, log, (err, userInfo) => { - if (err) { - log.debug('authentication error', { - error: err, - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, + return auth.server.doAuth( + request, + log, + (err, userInfo) => { + if (err) { + log.debug('authentication error', { + error: err, + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return responseJSONBody(err, null, response, log); + } + // FIXME for now, any authenticated user can access API + // routes. We should introduce admin accounts or accounts + // with admin privileges, and restrict access to those + // only. + if (userInfo.getCanonicalID() === constants.publicId) { + log.debug('unauthenticated access to API routes', { + method: request.method, + bucketName: request.bucketName, + objectKey: request.objectKey, + }); + return responseJSONBody(errors.AccessDenied, null, response, log); + } + return workflowEngineOperatorProxy.web(request, response, { target }, err => { + log.error('error proxying request to api server', { error: err.message }); + return responseJSONBody(errors.ServiceUnavailable, null, response, log); }); - return responseJSONBody(err, null, response, log); - } - // FIXME for now, any authenticated user can access API - // routes. We should introduce admin accounts or accounts - // with admin privileges, and restrict access to those - // only. - if (userInfo.getCanonicalID() === constants.publicId) { - log.debug('unauthenticated access to API routes', { - method: request.method, - bucketName: request.bucketName, - objectKey: request.objectKey, - }); - return responseJSONBody( - errors.AccessDenied, null, response, log); - } - return workflowEngineOperatorProxy.web( - request, response, { target }, err => { - log.error('error proxying request to api server', - { error: err.message }); - return responseJSONBody(errors.ServiceUnavailable, null, - response, log); - }); - }, 's3', requestContexts); + }, + 's3', + requestContexts, + ); } } - module.exports = routeWorkflowEngineOperator; diff --git a/lib/routes/veeam/delete.js b/lib/routes/veeam/delete.js index 20d73a1662..cfdba191c9 100644 --- a/lib/routes/veeam/delete.js +++ b/lib/routes/veeam/delete.js @@ -1,4 +1,3 @@ - const { s3routes, errors } = require('arsenal'); const metadata = require('../../metadata/wrapper'); const { isSystemXML } = require('./utils'); @@ -18,8 +17,7 @@ function deleteVeeamCapabilities(bucketName, objectKey, bucketMd, log, callback) const capabilityFieldName = isSystemXML(objectKey) ? 'SystemInfo' : 'CapacityInfo'; // Ensure file exists in metadata before deletion - if (!bucketMd._capabilities?.VeeamSOSApi - || !bucketMd._capabilities?.VeeamSOSApi[capabilityFieldName]) { + if (!bucketMd._capabilities?.VeeamSOSApi || !bucketMd._capabilities?.VeeamSOSApi[capabilityFieldName]) { return callback(errors.NoSuchKey); } // eslint-disable-next-line no-param-reassign diff --git a/lib/routes/veeam/get.js b/lib/routes/veeam/get.js index 213d5c4866..0159cd18b2 100644 --- a/lib/routes/veeam/get.js +++ b/lib/routes/veeam/get.js @@ -17,8 +17,13 @@ async function getVeeamFile(request, response, bucketMd, log) { } if ('tagging' in request.query) { - return await respondWithData(request, response, log, bucketMd, - buildHeadXML('')); + return await respondWithData( + request, + response, + log, + bucketMd, + buildHeadXML(''), + ); } try { diff --git a/lib/routes/veeam/list.js b/lib/routes/veeam/list.js index 175fbb77b4..a4a1bb5763 100644 --- a/lib/routes/veeam/list.js +++ b/lib/routes/veeam/list.js @@ -44,7 +44,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) { DisplayName: 'Veeam SOSAPI', }, StorageClass: 'VIRTUAL', - } + }, })); entries.push({ key: validPath, @@ -59,7 +59,7 @@ function buildXMLResponse(request, arrayOfFiles, versioned = false) { DisplayName: 'Veeam SOSAPI', }, StorageClass: 'VIRTUAL', - } + }, }); // Add the folder as the base file if (versioned) { @@ -87,8 +87,12 @@ async function listVeeamFiles(request, response, bucketMd, log) { // Only accept list-type query parameter if (!('list-type' in request.query) && !('versions' in request.query)) { - return responseXMLBody(errorInstances.InvalidRequest - .customizeDescription('The Veeam folder does not support this action.'), null, response, log); + return responseXMLBody( + errorInstances.InvalidRequest.customizeDescription('The Veeam folder does not support this action.'), + null, + response, + log, + ); } let data; @@ -125,22 +129,24 @@ async function listVeeamFiles(request, response, bucketMd, log) { } fieldsToGenerate.forEach(file => { const isCapacity = file.name.endsWith('capacity.xml'); - const lastModified = isCapacity - ? bucketMetrics.date - : file.LastModified; + const lastModified = isCapacity ? bucketMetrics.date : file.LastModified; // eslint-disable-next-line no-param-reassign delete file.LastModified; const dataBuffer = Buffer.from(buildXML(file)); filesToBuild.push({ - ...getResponseHeader(request, data, - dataBuffer, lastModified, log), + ...getResponseHeader(request, data, dataBuffer, lastModified, log), name: file.name, }); }); // When `versions` is present, listing should return a versioned list - return await respondWithData(request, response, log, data, - buildXMLResponse(request, filesToBuild, 'versions' in request.query)); + return await respondWithData( + request, + response, + log, + data, + buildXMLResponse(request, filesToBuild, 'versions' in request.query), + ); } module.exports = listVeeamFiles; diff --git a/lib/routes/veeam/put.js b/lib/routes/veeam/put.js index 0e15d86051..d9991775af 100644 --- a/lib/routes/veeam/put.js +++ b/lib/routes/veeam/put.js @@ -25,57 +25,68 @@ function putVeeamFile(request, response, bucketMd, log) { return errors.NoSuchBucket; } - return async.waterfall([ - next => { - // Extract the data from the request, keep it in memory - writeContinue(request, response); - return callbackify(receiveData)(request, log, next); - }, - (value, next) => parseString(value, { explicitArray: false }, (err, parsed) => { - // Convert the received XML to a JS object + return async.waterfall( + [ + next => { + // Extract the data from the request, keep it in memory + writeContinue(request, response); + return callbackify(receiveData)(request, log, next); + }, + (value, next) => + parseString(value, { explicitArray: false }, (err, parsed) => { + // Convert the received XML to a JS object + if (err) { + return next(errors.MalformedXML); + } + return next(null, parsed); + }), + (parsedXML, next) => { + const capabilities = bucketMd._capabilities || { + VeeamSOSApi: {}, + }; + // Validate the JS object schema with joi and prepare the object for + // further logic + const validateFn = isSystemXML(request.objectKey) ? parseSystemSchema : parseCapacitySchema; + let validatedData = null; + try { + validatedData = validateFn(parsedXML); + } catch (err) { + log.error('xml file did not pass validation', { err }); + return next(errors.MalformedXML); + } + const file = getFileToBuild(request, validatedData, true); + if (file.error) { + return next(file.error); + } + capabilities.VeeamSOSApi = { + ...(capabilities.VeeamSOSApi || {}), + ...file.value, + }; + // Write data to bucketMD with the same (validated) format + // eslint-disable-next-line no-param-reassign + bucketMd = { + ...bucketMd, + _capabilities: capabilities, + }; + // Update bucket metadata + return metadata.updateBucketCapabilities( + request.bucketName, + bucketMd, + 'VeeamSOSApi', + file.fieldName, + file.value[file.fieldName], + log, + next, + ); + }, + ], + err => { if (err) { - return next(errors.MalformedXML); - } - return next(null, parsed); - }), - (parsedXML, next) => { - const capabilities = bucketMd._capabilities || { - VeeamSOSApi: {}, - }; - // Validate the JS object schema with joi and prepare the object for - // further logic - const validateFn = isSystemXML(request.objectKey) ? parseSystemSchema : parseCapacitySchema; - let validatedData = null; - try { - validatedData = validateFn(parsedXML); - } catch (err) { - log.error('xml file did not pass validation', { err }); - return next(errors.MalformedXML); + return responseXMLBody(err, null, response, log); } - const file = getFileToBuild(request, validatedData, true); - if (file.error) { - return next(file.error); - } - capabilities.VeeamSOSApi = { - ...(capabilities.VeeamSOSApi || {}), - ...file.value, - }; - // Write data to bucketMD with the same (validated) format - // eslint-disable-next-line no-param-reassign - bucketMd = { - ...bucketMd, - _capabilities: capabilities, - }; - // Update bucket metadata - return metadata.updateBucketCapabilities( - request.bucketName, bucketMd, 'VeeamSOSApi', file.fieldName, file.value[file.fieldName], log, next); - } - ], err => { - if (err) { - return responseXMLBody(err, null, response, log); - } - return responseNoBody(null, null, response, 200, log); - }); + return responseNoBody(null, null, response, 200, log); + }, + ); } module.exports = putVeeamFile; diff --git a/lib/routes/veeam/schemas/capacity.js b/lib/routes/veeam/schemas/capacity.js index 3cd81f7b40..4897b1dae4 100644 --- a/lib/routes/veeam/schemas/capacity.js +++ b/lib/routes/veeam/schemas/capacity.js @@ -18,11 +18,13 @@ const { errors } = require('arsenal'); */ function validateCapacitySchema(parsedXML) { const schema = joi.object({ - CapacityInfo: joi.object({ - Capacity: joi.number().min(-1).integer().required(), - Available: joi.number().min(-1).integer().required(), - Used: joi.number().min(-1).integer().required(), - }).required(), + CapacityInfo: joi + .object({ + Capacity: joi.number().min(-1).integer().required(), + Available: joi.number().min(-1).integer().required(), + Used: joi.number().min(-1).integer().required(), + }) + .required(), }); const validatedData = schema.validate(parsedXML, { // Allow any unknown keys for future compatibility diff --git a/lib/routes/veeam/schemas/system.js b/lib/routes/veeam/schemas/system.js index 400bf22c32..183cf7fd06 100644 --- a/lib/routes/veeam/schemas/system.js +++ b/lib/routes/veeam/schemas/system.js @@ -3,29 +3,31 @@ const { errors, errorInstances } = require('arsenal'); // Allow supporting any version of the protocol const systemSchemasPerVersion = { - 'unsupported': joi.object({}), + unsupported: joi.object({}), '"1.0"': joi.object({ - SystemInfo: joi.object({ - ProtocolVersion: joi.string().required(), - ModelName: joi.string().required(), - ProtocolCapabilities: joi.object({ - CapacityInfo: joi.boolean().required(), - UploadSessions: joi.boolean().required(), - IAMSTS: joi.boolean().default(false), - }).required(), - APIEndpoints: joi.object({ - IAMEndpoint: joi.string().required(), - STSEndpoint: joi.string().required() - }), - SystemRecommendations: joi.object({ - S3ConcurrentTaskLimit: joi.number().min(0).default(64), - S3MultiObjectDeleteLimit: joi.number().min(1).default(1000), - StorageCurrentTasksLimit: joi.number().min(0).default(0), - KbBlockSize: joi.number() - .valid(256, 512, 1024, 2048, 4096, 8192) - .default(1024), - }), - }).required() + SystemInfo: joi + .object({ + ProtocolVersion: joi.string().required(), + ModelName: joi.string().required(), + ProtocolCapabilities: joi + .object({ + CapacityInfo: joi.boolean().required(), + UploadSessions: joi.boolean().required(), + IAMSTS: joi.boolean().default(false), + }) + .required(), + APIEndpoints: joi.object({ + IAMEndpoint: joi.string().required(), + STSEndpoint: joi.string().required(), + }), + SystemRecommendations: joi.object({ + S3ConcurrentTaskLimit: joi.number().min(0).default(64), + S3MultiObjectDeleteLimit: joi.number().min(1).default(1000), + StorageCurrentTasksLimit: joi.number().min(0).default(0), + KbBlockSize: joi.number().valid(256, 512, 1024, 2048, 4096, 8192).default(1024), + }), + }) + .required(), }), }; @@ -62,8 +64,9 @@ function validateSystemSchema(parsedXML) { const protocolVersion = parsedXML?.SystemInfo?.ProtocolVersion; let schema = systemSchemasPerVersion.unsupported; if (!protocolVersion) { - throw new Error(errorInstances.MalformedXML - .customizeDescription('ProtocolVersion must be set for the system.xml file')); + throw new Error( + errorInstances.MalformedXML.customizeDescription('ProtocolVersion must be set for the system.xml file'), + ); } if (protocolVersion && protocolVersion in systemSchemasPerVersion) { schema = systemSchemasPerVersion[parsedXML?.SystemInfo?.ProtocolVersion]; @@ -80,8 +83,10 @@ function validateSystemSchema(parsedXML) { case '"1.0"': // Ensure conditional fields are set // IAMSTS === true implies that SystemInfo.APIEndpoints is defined - if (validatedData.value.SystemInfo.ProtocolCapabilities.IAMSTS - && !validatedData.value.SystemInfo.APIEndpoints) { + if ( + validatedData.value.SystemInfo.ProtocolCapabilities.IAMSTS && + !validatedData.value.SystemInfo.APIEndpoints + ) { throw new Error(errors.MalformedXML); } break; diff --git a/lib/utapi/utapiReindex.js b/lib/utapi/utapiReindex.js index 9bc1522355..ad58f02e81 100644 --- a/lib/utapi/utapiReindex.js +++ b/lib/utapi/utapiReindex.js @@ -4,8 +4,7 @@ const { config } = require('../Config'); const reindexConfig = config.utapi && config.utapi.reindex; if (reindexConfig && reindexConfig.password === undefined) { - reindexConfig.password = config.utapi && config.utapi.redis - && config.utapi.redis.password; + reindexConfig.password = config.utapi && config.utapi.redis && config.utapi.redis.password; } const reindex = new UtapiReindex(reindexConfig); reindex.start(); diff --git a/lib/utapi/utilities.js b/lib/utapi/utilities.js index 32ea8432ac..061b283888 100644 --- a/lib/utapi/utilities.js +++ b/lib/utapi/utilities.js @@ -12,10 +12,13 @@ let utapiConfig; if (utapiVersion === 1 && _config.utapi) { utapiConfig = Object.assign({}, _config.utapi); } else if (utapiVersion === 2) { - utapiConfig = Object.assign({ - tls: _config.https, - suppressedEventFields, - }, _config.utapi || {}); + utapiConfig = Object.assign( + { + tls: _config.https, + suppressedEventFields, + }, + _config.utapi || {}, + ); } const utapi = new UtapiClient(utapiConfig); @@ -33,8 +36,7 @@ const bucketOwnerMetrics = [ function evalAuthInfo(authInfo, canonicalID, action) { let accountId = authInfo.getCanonicalID(); - let userId = authInfo.isRequesterAnIAMUser() ? - authInfo.getShortid() : undefined; + let userId = authInfo.isRequesterAnIAMUser() ? authInfo.getShortid() : undefined; // If action impacts 'numberOfObjectsStored' or 'storageUtilized' metric // only the bucket owner account's metrics should be updated const canonicalIdMatch = authInfo.getCanonicalID() === canonicalID; @@ -48,21 +50,10 @@ function evalAuthInfo(authInfo, canonicalID, action) { }; } -function _listMetrics(host, - port, - metric, - metricType, - timeRange, - accessKey, - secretKey, - verbose, - recent, - ssl) { +function _listMetrics(host, port, metric, metricType, timeRange, accessKey, secretKey, verbose, recent, ssl) { const listAction = recent ? 'ListRecentMetrics' : 'ListMetrics'; // If recent listing, we do not provide `timeRange` in the request - const requestObj = recent - ? { [metric]: metricType } - : { timeRange, [metric]: metricType }; + const requestObj = recent ? { [metric]: metricType } : { timeRange, [metric]: metricType }; const requestBody = JSON.stringify(requestObj); const options = { host, @@ -104,8 +95,7 @@ function _listMetrics(host, }); // TODO: cleanup with refactor of generateV4Headers request.path = `/${metric}`; - auth.client.generateV4Headers(request, { Action: listAction }, - accessKey, secretKey, 's3'); + auth.client.generateV4Headers(request, { Action: listAction }, accessKey, secretKey, 's3'); request.path = `/${metric}?Action=${listAction}`; if (verbose) { logger.info('request headers', { headers: request.getHeaders() }); @@ -132,24 +122,18 @@ function listMetrics(metricType) { // bin/list_bucket_metrics.js when prior method of listing bucket metrics is // no longer supported. if (metricType === 'buckets') { - program - .option('-b, --buckets ', 'Name of bucket(s) with ' + - 'a comma separator if more than one'); + program.option('-b, --buckets ', 'Name of bucket(s) with ' + 'a comma separator if more than one'); } else { program .option('-m, --metric ', 'Metric type') - .option('--buckets ', 'Name of bucket(s) with a comma ' + - 'separator if more than one') - .option('--accounts ', 'Account ID(s) with a comma ' + - 'separator if more than one') - .option('--users ', 'User ID(s) with a comma separator if ' + - 'more than one') + .option('--buckets ', 'Name of bucket(s) with a comma ' + 'separator if more than one') + .option('--accounts ', 'Account ID(s) with a comma ' + 'separator if more than one') + .option('--users ', 'User ID(s) with a comma separator if ' + 'more than one') .option('--service ', 'Name of service'); } program .option('-s, --start ', 'Start of time range') - .option('-r, --recent', 'List metrics including the previous and ' + - 'current 15 minute interval') + .option('-r, --recent', 'List metrics including the previous and ' + 'current 15 minute interval') .option('-e --end ', 'End of time range') .option('-h, --host ', 'Host of the server') .option('-p, --port ', 'Port of the server') @@ -157,7 +141,6 @@ function listMetrics(metricType) { .option('-v, --verbose') .parse(process.argv); - const providedOptions = program.opts(); const { host, port, accessKey, secretKey, start, end, verbose, recent, ssl, metric: metricLvl } = providedOptions; const requiredOptions = { host, port, accessKey, secretKey }; @@ -223,8 +206,7 @@ function listMetrics(metricType) { } } - _listMetrics(host, port, metric, resources, timeRange, accessKey, secretKey, - verbose, recent, ssl); + _listMetrics(host, port, metric, resources, timeRange, accessKey, secretKey, verbose, recent, ssl); } /** @@ -284,9 +266,11 @@ function pushMetric(action, log, metricObj) { let objectDelta = isDelete ? -numberOfObjects : numberOfObjects; // putDeleteMarkerObject does not pass numberOfObjects - if ((action === 'putDeleteMarkerObject' && byteLength === null) - || action === 'replicateDelete' - || action === 'replicateObject') { + if ( + (action === 'putDeleteMarkerObject' && byteLength === null) || + action === 'replicateDelete' || + action === 'replicateObject' + ) { objectDelta = 1; } else if (action === 'multiObjectDelete') { objectDelta = -(numberOfObjects + removedDeleteMarkers); @@ -303,8 +287,10 @@ function pushMetric(action, log, metricObj) { }; // Any operation from lifecycle that does not change object count or size is dropped - const isLifecycle = _config.lifecycleRoleName - && authInfo && authInfo.arn.endsWith(`:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`); + const isLifecycle = + _config.lifecycleRoleName && + authInfo && + authInfo.arn.endsWith(`:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`); if (isLifecycle && !objectDelta && !sizeDelta) { log.trace('ignoring pushMetric from lifecycle service user', { action, bucket, keys }); return undefined; @@ -381,8 +367,7 @@ function getLocationMetric(location, log, cb) { */ function pushLocationMetric(location, byteLength, log, cb) { const locationId = _getLocationId(location); - return utapi.pushLocationMetric(locationId, byteLength, - log.getSerializedUids(), cb); + return utapi.pushLocationMetric(locationId, byteLength, log.getSerializedUids(), cb); } module.exports = { diff --git a/lib/utilities/aclUtils.js b/lib/utilities/aclUtils.js index 7a3b820369..af9563174d 100644 --- a/lib/utilities/aclUtils.js +++ b/lib/utilities/aclUtils.js @@ -4,18 +4,18 @@ const { errors, s3middleware } = require('arsenal'); const constants = require('../../constants'); const escapeForXml = s3middleware.escapeForXml; -const possibleGrantHeaders = ['x-amz-grant-read', 'x-amz-grant-write', - 'x-amz-grant-read-acp', 'x-amz-grant-write-acp', - 'x-amz-grant-full-control']; +const possibleGrantHeaders = [ + 'x-amz-grant-read', + 'x-amz-grant-write', + 'x-amz-grant-read-acp', + 'x-amz-grant-write-acp', + 'x-amz-grant-full-control', +]; const regexpEmailAddress = /^\S+@\S+.\S+$/; const aclUtils = {}; -const grantsByURI = [ - constants.publicId, - constants.allAuthedUsersId, - constants.logId, -]; +const grantsByURI = [constants.publicId, constants.allAuthedUsersId, constants.logId]; /** * handleCannedGrant - Populate grantInfo for a bucketGetACL or objectGetACL @@ -26,86 +26,81 @@ const grantsByURI = [ * are different) * @returns {array} cannedGrants - containing canned ACL settings */ -aclUtils.handleCannedGrant = - function handleCannedGrant(grantType, - ownerGrant, separateBucketOwner) { - const cannedGrants = []; - const actions = { - 'private': () => { - cannedGrants.push(ownerGrant); - }, - 'public-read': () => { - const publicGrant = { - URI: constants.publicId, - permission: 'READ', - }; - cannedGrants.push(ownerGrant, publicGrant); - }, - 'public-read-write': () => { - const publicReadGrant = { - URI: constants.publicId, +aclUtils.handleCannedGrant = function handleCannedGrant(grantType, ownerGrant, separateBucketOwner) { + const cannedGrants = []; + const actions = { + private: () => { + cannedGrants.push(ownerGrant); + }, + 'public-read': () => { + const publicGrant = { + URI: constants.publicId, + permission: 'READ', + }; + cannedGrants.push(ownerGrant, publicGrant); + }, + 'public-read-write': () => { + const publicReadGrant = { + URI: constants.publicId, + permission: 'READ', + }; + const publicWriteGrant = { + URI: constants.publicId, + permission: 'WRITE', + }; + cannedGrants.push(ownerGrant, publicReadGrant, publicWriteGrant); + }, + 'authenticated-read': () => { + const authGrant = { + URI: constants.allAuthedUsersId, + permission: 'READ', + }; + cannedGrants.push(ownerGrant, authGrant); + }, + // Note: log-delivery-write is just for bucketGetACL + 'log-delivery-write': () => { + const logWriteGrant = { + URI: constants.logId, + permission: 'WRITE', + }; + const logReadACPGrant = { + URI: constants.logId, + permission: 'READ_ACP', + }; + cannedGrants.push(ownerGrant, logWriteGrant, logReadACPGrant); + }, + // Note: bucket-owner-read is just for objectGetACL + 'bucket-owner-read': () => { + // If the bucket owner and object owner are different, + // add separate entries for each + if (separateBucketOwner) { + const bucketOwnerReadGrant = { + ID: separateBucketOwner.getOwner(), + displayName: separateBucketOwner.getOwnerDisplayName(), permission: 'READ', }; - const publicWriteGrant = { - URI: constants.publicId, - permission: 'WRITE', - }; - cannedGrants. - push(ownerGrant, publicReadGrant, publicWriteGrant); - }, - 'authenticated-read': () => { - const authGrant = { - URI: constants.allAuthedUsersId, - permission: 'READ', - }; - cannedGrants.push(ownerGrant, authGrant); - }, - // Note: log-delivery-write is just for bucketGetACL - 'log-delivery-write': () => { - const logWriteGrant = { - URI: constants.logId, - permission: 'WRITE', - }; - const logReadACPGrant = { - URI: constants.logId, - permission: 'READ_ACP', + cannedGrants.push(ownerGrant, bucketOwnerReadGrant); + } else { + cannedGrants.push(ownerGrant); + } + }, + // Note: bucket-owner-full-control is just for objectGetACL + 'bucket-owner-full-control': () => { + if (separateBucketOwner) { + const bucketOwnerFCGrant = { + ID: separateBucketOwner.getOwner(), + displayName: separateBucketOwner.getOwnerDisplayName(), + permission: 'FULL_CONTROL', }; - cannedGrants. - push(ownerGrant, logWriteGrant, logReadACPGrant); - }, - // Note: bucket-owner-read is just for objectGetACL - 'bucket-owner-read': () => { - // If the bucket owner and object owner are different, - // add separate entries for each - if (separateBucketOwner) { - const bucketOwnerReadGrant = { - ID: separateBucketOwner.getOwner(), - displayName: separateBucketOwner.getOwnerDisplayName(), - permission: 'READ', - }; - cannedGrants.push(ownerGrant, bucketOwnerReadGrant); - } else { - cannedGrants.push(ownerGrant); - } - }, - // Note: bucket-owner-full-control is just for objectGetACL - 'bucket-owner-full-control': () => { - if (separateBucketOwner) { - const bucketOwnerFCGrant = { - ID: separateBucketOwner.getOwner(), - displayName: separateBucketOwner.getOwnerDisplayName(), - permission: 'FULL_CONTROL', - }; - cannedGrants.push(ownerGrant, bucketOwnerFCGrant); - } else { - cannedGrants.push(ownerGrant); - } - }, - }; - actions[grantType](); - return cannedGrants; + cannedGrants.push(ownerGrant, bucketOwnerFCGrant); + } else { + cannedGrants.push(ownerGrant); + } + }, }; - + actions[grantType](); + return cannedGrants; +}; aclUtils.parseAclXml = function parseAclXml(toBeParsed, log, next) { return parseString(toBeParsed, (err, result) => { @@ -113,24 +108,26 @@ aclUtils.parseAclXml = function parseAclXml(toBeParsed, log, next) { log.debug('invalid xml', { xmlObj: toBeParsed }); return next(errors.MalformedXML); } - if (!result.AccessControlPolicy - || !result.AccessControlPolicy.AccessControlList - || result.AccessControlPolicy.AccessControlList.length !== 1 - || (result.AccessControlPolicy.AccessControlList[0] !== '' && - Object.keys(result.AccessControlPolicy.AccessControlList[0]) - .some(listKey => listKey !== 'Grant'))) { + if ( + !result.AccessControlPolicy || + !result.AccessControlPolicy.AccessControlList || + result.AccessControlPolicy.AccessControlList.length !== 1 || + (result.AccessControlPolicy.AccessControlList[0] !== '' && + Object.keys(result.AccessControlPolicy.AccessControlList[0]).some(listKey => listKey !== 'Grant')) + ) { log.debug('invalid acl', { acl: result }); return next(errors.MalformedACLError); } - const jsonGrants = result - .AccessControlPolicy.AccessControlList[0].Grant; + const jsonGrants = result.AccessControlPolicy.AccessControlList[0].Grant; log.trace('acl grants', { aclGrants: jsonGrants }); - if (!Array.isArray(result.AccessControlPolicy.Owner) - || result.AccessControlPolicy.Owner.length !== 1 - || !Array.isArray(result.AccessControlPolicy.Owner[0].ID) - || result.AccessControlPolicy.Owner[0].ID.length !== 1 - || result.AccessControlPolicy.Owner[0].ID[0] === '') { + if ( + !Array.isArray(result.AccessControlPolicy.Owner) || + result.AccessControlPolicy.Owner.length !== 1 || + !Array.isArray(result.AccessControlPolicy.Owner[0].ID) || + result.AccessControlPolicy.Owner[0].ID.length !== 1 || + result.AccessControlPolicy.Owner[0].ID[0] === '' + ) { return next(errors.MalformedACLError); } const ownerID = result.AccessControlPolicy.Owner[0].ID[0]; @@ -139,8 +136,7 @@ aclUtils.parseAclXml = function parseAclXml(toBeParsed, log, next) { }); }; -aclUtils.getPermissionType = function getPermissionType(identifier, resourceACL, - resourceType) { +aclUtils.getPermissionType = function getPermissionType(identifier, resourceACL, resourceType) { const fullControlIndex = resourceACL.FULL_CONTROL.indexOf(identifier); let writeIndex; if (resourceType === 'bucket') { @@ -196,32 +192,29 @@ aclUtils.isValidCanonicalId = function isValidCanonicalId(canonicalID) { return /^(?=.*?[a-f])(?=.*?[0-9])[a-f0-9]{64}$/.test(canonicalID); }; -aclUtils.reconstructUsersIdentifiedByEmail = - function reconstruct(userInfofromVault, userGrantInfo) { - return userGrantInfo.map(item => { - const userEmail = item.identifier.toLowerCase(); - const user = {}; - // Find the full user grant info based on email - const userId = userInfofromVault - .find(elem => elem.email.toLowerCase() === userEmail); - // Set the identifier to be the canonicalID instead of email - user.identifier = userId.canonicalID; - user.userIDType = 'id'; - // copy over ACL grant type: i.e. READ/WRITE... - user.grantType = item.grantType; - return user; - }); - }; +aclUtils.reconstructUsersIdentifiedByEmail = function reconstruct(userInfofromVault, userGrantInfo) { + return userGrantInfo.map(item => { + const userEmail = item.identifier.toLowerCase(); + const user = {}; + // Find the full user grant info based on email + const userId = userInfofromVault.find(elem => elem.email.toLowerCase() === userEmail); + // Set the identifier to be the canonicalID instead of email + user.identifier = userId.canonicalID; + user.userIDType = 'id'; + // copy over ACL grant type: i.e. READ/WRITE... + user.grantType = item.grantType; + return user; + }); +}; -aclUtils.sortHeaderGrants = - function sortHeaderGrants(allGrantHeaders, addACLParams) { - allGrantHeaders.forEach(item => { - if (item) { - addACLParams[item.grantType].push(item.identifier); - } - }); - return addACLParams; - }; +aclUtils.sortHeaderGrants = function sortHeaderGrants(allGrantHeaders, addACLParams) { + allGrantHeaders.forEach(item => { + if (item) { + addACLParams[item.grantType].push(item.identifier); + } + }); + return addACLParams; +}; /** * convertToXml - Converts the `grantInfo` object (defined in `objectGetACL()`) @@ -234,14 +227,14 @@ aclUtils.convertToXml = grantInfo => { const { grants, ownerInfo } = grantInfo; const xml = []; - xml.push('', + xml.push( + '', '', '', `${ownerInfo.ID}`, - `${escapeForXml(ownerInfo.displayName)}` + - '', + `${escapeForXml(ownerInfo.displayName)}` + '', '', - '' + '', ); grants.forEach(grant => { @@ -250,32 +243,25 @@ aclUtils.convertToXml = grantInfo => { // The `` tag has different attributes depending on whether the // grant has an ID or URI if (grant.ID) { - xml.push('', - `${grant.ID}` + xml.push( + '', + `${grant.ID}`, ); } else if (grant.URI) { - xml.push('', - `${escapeForXml(grant.URI)}` + xml.push( + '', + `${escapeForXml(grant.URI)}`, ); } if (grant.displayName) { - xml.push(`${escapeForXml(grant.displayName)}` + - '' - ); + xml.push(`${escapeForXml(grant.displayName)}` + ''); } - xml.push('', - `${grant.permission}`, - '' - ); + xml.push('', `${grant.permission}`, ''); }); - xml.push('', - '' - ); + xml.push('', ''); return xml.join(''); }; @@ -303,9 +289,11 @@ aclUtils.checkGrantHeaderValidity = function checkGrantHeaderValidity(headers) { const identifier = singleGrantArr[0].trim().toLowerCase(); const value = singleGrantArr[1].trim(); if (identifier === 'uri') { - if (value !== constants.publicId && + if ( + value !== constants.publicId && value !== constants.allAuthedUsersId && - value !== constants.logId) { + value !== constants.logId + ) { return false; } } else if (identifier === 'emailaddress') { @@ -346,13 +334,7 @@ function getGrants(acl) { * @returns {array} canonicalIDs - array of unique canonicalIDs from acl */ aclUtils.getCanonicalIDs = function getCanonicalIDs(acl) { - const aclGrantees = [].concat( - acl.FULL_CONTROL, - acl.WRITE, - acl.WRITE_ACP, - acl.READ, - acl.READ_ACP - ); + const aclGrantees = [].concat(acl.FULL_CONTROL, acl.WRITE, acl.WRITE_ACP, acl.READ, acl.READ_ACP); const uniqueGrantees = Array.from(new Set(aclGrantees)); // grantees can be a mix of canonicalIDs and predefined groups in the form // of uri, so filter out only canonicalIDs @@ -367,11 +349,12 @@ aclUtils.getCanonicalIDs = function getCanonicalIDs(acl) { aclUtils.getUriGrantInfo = function getUriGrantInfo(acl) { const grants = getGrants(acl); const uriGrantInfo = []; - const validGrants = Object.entries(grants) - .filter(([permission, grantees]) => permission - && Array.isArray(grantees)); + const validGrants = Object.entries(grants).filter( + ([permission, grantees]) => permission && Array.isArray(grantees), + ); validGrants.forEach(([permission, grantees]) => { - grantees.filter(grantee => grantsByURI.includes(grantee)) + grantees + .filter(grantee => grantsByURI.includes(grantee)) .forEach(grantee => { uriGrantInfo.push({ URI: grantee, @@ -391,16 +374,15 @@ aclUtils.getUriGrantInfo = function getUriGrantInfo(acl) { * @returns {array} individualGrantInfo - array of grants mapped to * canonicalID/email */ -aclUtils.getIndividualGrants = function getIndividualGrants(acl, canonicalIDs, - emails) { +aclUtils.getIndividualGrants = function getIndividualGrants(acl, canonicalIDs, emails) { const grants = getGrants(acl); const individualGrantInfo = []; - const validGrants = Object.entries(grants) - .filter(([permission, grantees]) => permission - && Array.isArray(grantees)); + const validGrants = Object.entries(grants).filter( + ([permission, grantees]) => permission && Array.isArray(grantees), + ); validGrants.forEach(([permission, grantees]) => { - grantees.filter(grantee => canonicalIDs.includes(grantee) - && emails[grantee]) + grantees + .filter(grantee => canonicalIDs.includes(grantee) && emails[grantee]) .forEach(grantee => { individualGrantInfo.push({ ID: grantee, diff --git a/lib/utilities/collectCorsHeaders.js b/lib/utilities/collectCorsHeaders.js index 286e510953..a2e655c0db 100644 --- a/lib/utilities/collectCorsHeaders.js +++ b/lib/utilities/collectCorsHeaders.js @@ -1,5 +1,4 @@ -const { findCorsRule, generateCorsResHeaders } = - require('../api/apiUtils/object/corsResponse.js'); +const { findCorsRule, generateCorsResHeaders } = require('../api/apiUtils/object/corsResponse.js'); /** * collectCorsHeaders - gather any relevant CORS headers diff --git a/lib/utilities/healthcheckHandler.js b/lib/utilities/healthcheckHandler.js index 35186299f8..58e29d0257 100644 --- a/lib/utilities/healthcheckHandler.js +++ b/lib/utilities/healthcheckHandler.js @@ -41,12 +41,7 @@ function writeResponse(res, error, log, results, cb) { * @return {undefined} */ function clientCheck(flightCheckOnStartUp, log, cb) { - const clients = [ - data, - metadata, - vault, - kms, - ]; + const clients = [data, metadata, vault, kms]; const clientTasks = []; clients.forEach(client => { if (typeof client.checkHealth === 'function') { @@ -65,21 +60,27 @@ function clientCheck(flightCheckOnStartUp, log, cb) { // Each result in the array represents one client (data, metadata, vault, kms) // with its backend locations as keys. // If ALL backend/locations of ANY client have errors, the overall check fails. - const { obj, fail } = results.reduce((acc, clientResult) => { - Object.assign(acc.obj, clientResult); + const { obj, fail } = results.reduce( + (acc, clientResult) => { + Object.assign(acc.obj, clientResult); - // Check if ALL backends/locations of this client have errors - const keys = Object.keys(clientResult); - // eslint-disable-next-line no-param-reassign - acc.fail ||= keys.length > 0 && keys.every(k => - // if there is an error from an external backend, - // only return a 500 if it is on startup - // (flightCheckOnStartUp set to true) - clientResult[k].error && (flightCheckOnStartUp || !clientResult[k].external) - ); + // Check if ALL backends/locations of this client have errors + const keys = Object.keys(clientResult); + // eslint-disable-next-line no-param-reassign + acc.fail ||= + keys.length > 0 && + keys.every( + k => + // if there is an error from an external backend, + // only return a 500 if it is on startup + // (flightCheckOnStartUp set to true) + clientResult[k].error && (flightCheckOnStartUp || !clientResult[k].external), + ); - return acc; - }, { obj: {}, fail: false }); + return acc; + }, + { obj: {}, fail: false }, + ); if (fail) { return cb(errors.InternalError, obj); } @@ -106,8 +107,7 @@ function routeHandler(deep, req, res, log, statsClient, cb) { } function checkIP(clientIP) { - return ipCheck.ipMatchCidrList( - _config.healthChecks.allowFrom, clientIP); + return ipCheck.ipMatchCidrList(_config.healthChecks.allowFrom, clientIP); } /** @@ -139,8 +139,7 @@ function healthcheckHandler(clientIP, req, res, log, statsClient, deep) { if (!checkIP(clientIP)) { return healthcheckEndHandler(errors.AccessDenied, []); } - return routeHandler(deep, req, res, log, statsClient, - healthcheckEndHandler); + return routeHandler(deep, req, res, log, statsClient, healthcheckEndHandler); } module.exports = { diff --git a/lib/utilities/internalHandlers.js b/lib/utilities/internalHandlers.js index d3379158c0..c52d790489 100644 --- a/lib/utilities/internalHandlers.js +++ b/lib/utilities/internalHandlers.js @@ -1,7 +1,6 @@ const { routeBackbeat } = require('../routes/routeBackbeat'); const routeMetadata = require('../routes/routeMetadata'); -const routeWorkflowEngineOperator = - require('../routes/routeWorkflowEngineOperator'); +const routeWorkflowEngineOperator = require('../routes/routeWorkflowEngineOperator'); const { reportHandler } = require('./reportHandler'); const routeVeeam = require('../routes/routeVeeam').routeVeeam; const { healthcheckHandler } = require('./healthcheckHandler'); diff --git a/lib/utilities/legacyAWSBehavior.js b/lib/utilities/legacyAWSBehavior.js index c8a457a687..859d064972 100644 --- a/lib/utilities/legacyAWSBehavior.js +++ b/lib/utilities/legacyAWSBehavior.js @@ -10,8 +10,10 @@ const { config } = require('../Config'); */ function isLegacyAwsBehavior(locationConstraint) { - return (config.locationConstraints[locationConstraint] && - config.locationConstraints[locationConstraint].legacyAwsBehavior); + return ( + config.locationConstraints[locationConstraint] && + config.locationConstraints[locationConstraint].legacyAwsBehavior + ); } module.exports = isLegacyAwsBehavior; diff --git a/lib/utilities/request.js b/lib/utilities/request.js index 7be85927a6..7b5b09e4af 100644 --- a/lib/utilities/request.js +++ b/lib/utilities/request.js @@ -5,9 +5,7 @@ const { HttpProxyAgent } = require('http-proxy-agent'); const { HttpsProxyAgent } = require('https-proxy-agent'); const { jsutil } = require('arsenal'); -const { - proxyCompareUrl, -} = require('arsenal').storage.data.external.backendUtils; +const { proxyCompareUrl } = require('arsenal').storage.data.external.backendUtils; const validVerbs = new Set(['HEAD', 'GET', 'POST', 'PUT', 'DELETE']); const updateVerbs = new Set(['POST', 'PUT']); @@ -16,7 +14,7 @@ const updateVerbs = new Set(['POST', 'PUT']); * create a new header object from an existing header object. Similar keys * will be ignored if a value has been set for theirlower-cased form */ -function createHeaders(headers) { +function createHeaders(headers) { if (typeof headers !== 'object') { return {}; } @@ -72,7 +70,7 @@ function request(endpoint, options, callback) { let reqParams; if (typeof endpoint === 'string') { try { - reqParams = url.parse(endpoint); + reqParams = url.parse(endpoint); } catch (error) { return cb(error); } @@ -115,8 +113,10 @@ function request(endpoint, options, callback) { const req = request.request(reqParams); req.on('error', cb); req.on('response', res => { - const rawData = []; - res.on('data', chunk => { rawData.push(chunk); }); + const rawData = []; + res.on('data', chunk => { + rawData.push(chunk); + }); res.on('end', () => { const data = rawData.join(''); if (res.statusCode >= 400) { diff --git a/lib/utilities/serverAccessLogger.js b/lib/utilities/serverAccessLogger.js index 06eae55883..98c3526f22 100644 --- a/lib/utilities/serverAccessLogger.js +++ b/lib/utilities/serverAccessLogger.js @@ -17,7 +17,9 @@ class ServerAccessLogger { this._terminated = false; this._reopenStream(); - setInterval(() => { this._checkFileRotated(); }, checkFileRotationIntervalMS); + setInterval(() => { + this._checkFileRotated(); + }, checkFileRotationIntervalMS); process.on('beforeExit', () => { this._cleanOldStream(true); @@ -48,7 +50,7 @@ class ServerAccessLogger { if (err) { if (err.code === 'ENOENT') { - logger.info('ServerAccessLogger: log file doesn\'t exist, creating a new one'); + logger.info("ServerAccessLogger: log file doesn't exist, creating a new one"); this._reopenStream(); return; } @@ -86,7 +88,9 @@ class ServerAccessLogger { fs.open(this._filename, 'a', 0o644, (err, fd) => { if (err) { logger.error('ServerAccessLogger: failed to reopenStream: open error', err); - setTimeout(() => { this._reopenStream(); }, this._retryReopenDelayMS); + setTimeout(() => { + this._reopenStream(); + }, this._retryReopenDelayMS); return; } @@ -100,7 +104,9 @@ class ServerAccessLogger { logger.error('ServerAccessLogger: failed to reopenStream: stat error', err); fs.close(fd); - setTimeout(() => { this._reopenStream(); }, this._retryReopenDelayMS); + setTimeout(() => { + this._reopenStream(); + }, this._retryReopenDelayMS); return; } @@ -144,10 +150,12 @@ class ServerAccessLogger { if (!this.stream.write(data)) { logger.warn('ServerAccessLogger: backpressure buffer full'); this._waitingDrain = true; - this.stream.once('drain', () => { this._waitingDrain = false; }); + this.stream.once('drain', () => { + this._waitingDrain = false; + }); } } -}; +} function setServerAccessLogger(logger) { serverAccessLogger = logger; @@ -183,7 +191,8 @@ function getRemoteIPFromRequest(request) { let remoteIP = null; if (request.headers) { // Check for forwarded IP headers (proxy/load balancer scenarios) - const headerRemoteIP = request.headers['x-forwarded-for'] || + const headerRemoteIP = + request.headers['x-forwarded-for'] || request.headers['x-real-ip'] || request.headers['x-client-ip'] || request.headers['cf-connecting-ip']; // Cloudflare @@ -196,9 +205,10 @@ function getRemoteIPFromRequest(request) { // Fallback to connection remote address if no forwarded headers if (!remoteIP) { - const connIP = (request.connection && request.connection.remoteAddress) || + const connIP = + (request.connection && request.connection.remoteAddress) || (request.socket && request.socket.remoteAddress) || - (request.ip); + request.ip; if (connIP) { remoteIP = connIP; } @@ -207,82 +217,81 @@ function getRemoteIPFromRequest(request) { return remoteIP; } -// eslint-disable-next-line max-len // https://github.com/awslabs/glue-extensions-for-iceberg/blob/52bdb2908216a85859fd76a45981d0326d016a2f/spark/src/main/scala/software/amazon/glue/s3a/audit/S3LogVerbs.java // https://github.com/open-io/swift/blob/ff518e9907f74b5a2565973a260f36386b5d5cbf/etc/s3-default.cfg.in#L78 // https://stackoverflow.com/questions/42707878/amazon-s3-logs-operation-definition const methodToResType = Object.freeze({ - 'bucketDelete': 'BUCKET', - 'bucketDeleteCors': 'CORS', - 'bucketDeleteEncryption': 'ENCRYPTION', - 'bucketDeleteWebsite': 'WEBSITE', - 'bucketGet': 'BUCKET', - 'bucketGetACL': 'ACL', - 'bucketGetCors': 'CORS', - 'bucketGetObjectLock': 'OBJECT', - 'bucketGetVersioning': 'VERSIONING', - 'bucketGetWebsite': 'WEBSITE', - 'bucketGetLocation': 'LOCATION', - 'bucketGetEncryption': 'ENCRYPTION', - 'bucketHead': 'BUCKET', - 'bucketPut': 'BUCKET', - 'bucketPutACL': 'ACL', - 'bucketPutCors': 'CORS', - 'bucketPutVersioning': 'VERSIONING', - 'bucketPutTagging': 'TAGGING', - 'bucketDeleteTagging': 'TAGGING', - 'bucketGetTagging': 'TAGGING', - 'bucketPutWebsite': 'WEBSITE', - 'bucketPutReplication': 'REPLICATION', - 'bucketGetReplication': 'REPLICATION', - 'bucketDeleteReplication': 'REPLICATION', - 'bucketDeleteQuota': 'QUOTA', - 'bucketPutLifecycle': 'LIFECYCLE', - 'bucketUpdateQuota': 'QUOTA', - 'bucketGetLifecycle': 'LIFECYCLE', - 'bucketDeleteLifecycle': 'LIFECYCLE', - 'bucketPutPolicy': 'BUCKETPOLICY', - 'bucketGetPolicy': 'BUCKETPOLICY', - 'bucketGetQuota': 'QUOTA', - 'bucketDeletePolicy': 'BUCKETPOLICY', - 'bucketPutObjectLock': 'OBJECT', - 'bucketPutNotification': 'NOTIFICATION', - 'bucketGetNotification': 'NOTIFICATION', - 'bucketPutEncryption': 'ENCRYPTION', - 'bucketPutLogging': 'LOGGING_STATUS', - 'bucketGetLogging': 'LOGGING_STATUS', - 'bucketPutRateLimit': 'RATELIMIT', - 'bucketGetRateLimit': 'RATELIMIT', - 'bucketDeleteRateLimit': 'RATELIMIT', - 'corsPreflight': 'PREFLIGHT', - 'completeMultipartUpload': 'UPLOAD', - 'initiateMultipartUpload': 'UPLOADS', - 'listMultipartUploads': 'UPLOADS', - 'listParts': 'UPLOAD', - 'metadataSearch': 'OBJECT', - 'multiObjectDelete': 'MULTI_OBJECT_DELETE', - 'multipartDelete': 'UPLOAD', - 'objectDelete': 'OBJECT', - 'objectDeleteTagging': 'TAGGING', - 'objectGet': 'OBJECT', - 'objectGetAttributes': 'OBJECT', - 'objectGetACL': 'ACL', - 'objectGetLegalHold': 'LEGALHOLD', - 'objectGetRetention': 'OBJECT_LOCK_RETENTION', - 'objectGetTagging': 'TAGGING', - 'objectCopy': 'COPY', - 'objectHead': 'OBJECT', - 'objectPut': 'OBJECT', - 'objectPutACL': 'ACL', - 'objectPutLegalHold': 'LEGALHOLD', - 'objectPutTagging': 'TAGGING', - 'objectPutPart': 'PART', - 'objectPutCopyPart': 'COPY', - 'objectPutRetention': 'OBJECT_LOCK_RETENTION', - 'objectRestore': 'OBJECT', - 'serviceGet': 'SERVICE', // ListBuckets - 'websiteGet': 'WEBSITE', - 'websiteHead': 'WEBSITE', + bucketDelete: 'BUCKET', + bucketDeleteCors: 'CORS', + bucketDeleteEncryption: 'ENCRYPTION', + bucketDeleteWebsite: 'WEBSITE', + bucketGet: 'BUCKET', + bucketGetACL: 'ACL', + bucketGetCors: 'CORS', + bucketGetObjectLock: 'OBJECT', + bucketGetVersioning: 'VERSIONING', + bucketGetWebsite: 'WEBSITE', + bucketGetLocation: 'LOCATION', + bucketGetEncryption: 'ENCRYPTION', + bucketHead: 'BUCKET', + bucketPut: 'BUCKET', + bucketPutACL: 'ACL', + bucketPutCors: 'CORS', + bucketPutVersioning: 'VERSIONING', + bucketPutTagging: 'TAGGING', + bucketDeleteTagging: 'TAGGING', + bucketGetTagging: 'TAGGING', + bucketPutWebsite: 'WEBSITE', + bucketPutReplication: 'REPLICATION', + bucketGetReplication: 'REPLICATION', + bucketDeleteReplication: 'REPLICATION', + bucketDeleteQuota: 'QUOTA', + bucketPutLifecycle: 'LIFECYCLE', + bucketUpdateQuota: 'QUOTA', + bucketGetLifecycle: 'LIFECYCLE', + bucketDeleteLifecycle: 'LIFECYCLE', + bucketPutPolicy: 'BUCKETPOLICY', + bucketGetPolicy: 'BUCKETPOLICY', + bucketGetQuota: 'QUOTA', + bucketDeletePolicy: 'BUCKETPOLICY', + bucketPutObjectLock: 'OBJECT', + bucketPutNotification: 'NOTIFICATION', + bucketGetNotification: 'NOTIFICATION', + bucketPutEncryption: 'ENCRYPTION', + bucketPutLogging: 'LOGGING_STATUS', + bucketGetLogging: 'LOGGING_STATUS', + bucketPutRateLimit: 'RATELIMIT', + bucketGetRateLimit: 'RATELIMIT', + bucketDeleteRateLimit: 'RATELIMIT', + corsPreflight: 'PREFLIGHT', + completeMultipartUpload: 'UPLOAD', + initiateMultipartUpload: 'UPLOADS', + listMultipartUploads: 'UPLOADS', + listParts: 'UPLOAD', + metadataSearch: 'OBJECT', + multiObjectDelete: 'MULTI_OBJECT_DELETE', + multipartDelete: 'UPLOAD', + objectDelete: 'OBJECT', + objectDeleteTagging: 'TAGGING', + objectGet: 'OBJECT', + objectGetAttributes: 'OBJECT', + objectGetACL: 'ACL', + objectGetLegalHold: 'LEGALHOLD', + objectGetRetention: 'OBJECT_LOCK_RETENTION', + objectGetTagging: 'TAGGING', + objectCopy: 'COPY', + objectHead: 'OBJECT', + objectPut: 'OBJECT', + objectPutACL: 'ACL', + objectPutLegalHold: 'LEGALHOLD', + objectPutTagging: 'TAGGING', + objectPutPart: 'PART', + objectPutCopyPart: 'COPY', + objectPutRetention: 'OBJECT_LOCK_RETENTION', + objectRestore: 'OBJECT', + serviceGet: 'SERVICE', // ListBuckets + websiteGet: 'WEBSITE', + websiteHead: 'WEBSITE', }); function getOperation(req) { @@ -330,7 +339,7 @@ function getOperation(req) { process.emitWarning('Unknown apiMethod for server access log', { type: 'ServerAccessLogWarning', code: 'UNKNOWN_API_METHOD', - detail: `apiMethod=${req.apiMethod}, method=${req.method}, url=${req.url}` + detail: `apiMethod=${req.apiMethod}, method=${req.method}, url=${req.url}`, }); } return `REST.${req.method}.UNKNOWN`; @@ -374,12 +383,12 @@ function getURI(request) { } const objectSizePutMethods = Object.freeze({ - 'objectPut': true, - 'objectPutPart': true, + objectPut: true, + objectPutPart: true, }); const objectSizeGetMethods = Object.freeze({ - 'objectGet': true, + objectGet: true, }); function getObjectSize(request, response) { @@ -441,8 +450,12 @@ function calculateTotalTime(startTime, onFinishEndTime) { } function calculateTurnAroundTime(startTurnAroundTime, endTurnAroundTime) { - if (startTurnAroundTime === undefined || startTurnAroundTime === null - || endTurnAroundTime === undefined || endTurnAroundTime === null) { + if ( + startTurnAroundTime === undefined || + startTurnAroundTime === null || + endTurnAroundTime === undefined || + endTurnAroundTime === null + ) { return null; } @@ -510,12 +523,12 @@ function buildLogEntry(req, params, options) { signatureVersion: authInfo?.getAuthVersion() ?? undefined, cipherSuite: req.socket?.encrypted ? req.socket.getCipher()['standardName'] - : req.headers?.['x-ssl-cipher'] ?? undefined, + : (req.headers?.['x-ssl-cipher'] ?? undefined), authenticationType: authInfo?.getAuthType() ?? undefined, hostHeader: req.headers?.host ?? undefined, tlsVersion: req.socket?.encrypted ? req.socket.getCipher()['version'] - : req.headers?.['x-ssl-protocol'] ?? undefined, + : (req.headers?.['x-ssl-protocol'] ?? undefined), aclRequired: options.aclRequired ?? undefined, // hostID: undefined, // NOT IMPLEMENTED // accessPointARN: undefined, // NOT IMPLEMENTED @@ -526,7 +539,7 @@ function buildLogEntry(req, params, options) { // eslint-disable-next-line camelcase req_id: options.requestID ?? undefined, // AWS "Request ID" field bytesSent: options.bytesSent ?? undefined, - clientIP: getRemoteIPFromRequest(req) ?? undefined, // AWS 'Remote IP' field + clientIP: getRemoteIPFromRequest(req) ?? undefined, // AWS 'Remote IP' field httpCode: options.httpCode ?? undefined, // AWS "HTTP Status" field objectKey: options.objectKey ?? undefined, // AWS "Key" field @@ -534,8 +547,8 @@ function buildLogEntry(req, params, options) { logFormatVersion: SERVER_ACCESS_LOG_FORMAT_VERSION, // For backbeat requests other than expiration and replication, // force loggingEnabled to false to prevent delivery to log courier. - loggingEnabled: (params.backbeat && !params.expiration && !params.replication) - ? false : (params.enabled ?? undefined), + loggingEnabled: + params.backbeat && !params.expiration && !params.replication ? false : (params.enabled ?? undefined), loggingTargetBucket: params.loggingEnabled?.TargetBucket ?? undefined, loggingTargetPrefix: params.loggingEnabled?.TargetPrefix ?? undefined, awsAccessKeyID: authInfo?.getAccessKey() ?? undefined, @@ -599,9 +612,8 @@ function logServerAccess(req, res) { const logEntry = buildLogEntry(req, params, { bytesDeleted: params.analyticsBytesDeleted, bytesReceived: Number.isInteger(req.parsedContentLength) ? req.parsedContentLength : undefined, - bodyLength: req.headers['content-length'] !== undefined - ? parseInt(req.headers['content-length'], 10) - : undefined, + bodyLength: + req.headers['content-length'] !== undefined ? parseInt(req.headers['content-length'], 10) : undefined, contentLength: getObjectSize(req, res), // eslint-disable-next-line camelcase elapsed_ms: calculateElapsedMS(params.startTime, params.onCloseEndTime), @@ -670,8 +682,7 @@ function logServerAccess(req, res) { query = versionId ? `?acl&versionId=${versionId}` : '?acl'; } const encodedKey = params.objectKey.split('/').map(encodeURIComponent).join('/'); - logEntry.requestURI = - `${method} /${params.bucketName}/${encodedKey}${query} HTTP/${req.httpVersion ?? '1.1'}`; + logEntry.requestURI = `${method} /${params.bucketName}/${encodedKey}${query} HTTP/${req.httpVersion ?? '1.1'}`; } if (params.internalLogRequestQueue && params.internalLogRequestQueue.length > 0) { @@ -681,8 +692,14 @@ function logServerAccess(req, res) { } } else if (logEntry.operation.includes('COPY')) { for (const entry of params.internalLogRequestQueue) { - logCopySourceAccess(req, requestID, entry.operation, entry.sourceBucket, - entry.sourceObject, entry.objectSize); + logCopySourceAccess( + req, + requestID, + entry.operation, + entry.sourceBucket, + entry.sourceObject, + entry.objectSize, + ); } } } diff --git a/lib/utilities/validateQueryAndHeaders.js b/lib/utilities/validateQueryAndHeaders.js index 376e3c7c24..37c07727ba 100644 --- a/lib/utilities/validateQueryAndHeaders.js +++ b/lib/utilities/validateQueryAndHeaders.js @@ -14,7 +14,6 @@ function _validateKeys(unsupportedKeys, obj) { return unsupportedKey; } - /** * validateQueryAndHeaders - Check request for unsupported queries or headers * @param {object} request - request object @@ -28,8 +27,7 @@ function validateQueryAndHeaders(request, log) { const isBucketQuery = !request.objectKey; // if the request is at bucket level, check for unsupported bucket queries if (isBucketQuery) { - const unsupportedQuery = - _validateKeys(constants.unsupportedBucketQueries, reqQuery); + const unsupportedQuery = _validateKeys(constants.unsupportedBucketQueries, reqQuery); if (unsupportedQuery) { log.debug('encountered unsupported query', { query: unsupportedQuery, @@ -38,8 +36,7 @@ function validateQueryAndHeaders(request, log) { return { error: errors.NotImplemented }; } } - const unsupportedQuery = _validateKeys(constants.unsupportedQueries, - reqQuery); + const unsupportedQuery = _validateKeys(constants.unsupportedQueries, reqQuery); if (unsupportedQuery) { log.debug('encountered unsupported query', { query: unsupportedQuery, @@ -47,8 +44,7 @@ function validateQueryAndHeaders(request, log) { }); return { error: errors.NotImplemented }; } - const unsupportedHeader = _validateKeys(constants.unsupportedHeaders, - reqHeaders); + const unsupportedHeader = _validateKeys(constants.unsupportedHeaders, reqHeaders); if (unsupportedHeader) { log.debug('encountered unsupported header', { header: unsupportedHeader, diff --git a/managementAgent.js b/managementAgent.js index d7161ef693..4413d36c3b 100644 --- a/managementAgent.js +++ b/managementAgent.js @@ -8,13 +8,11 @@ const { managementAgentMessageType } = require('./lib/management/agentClient'); const { addOverlayMessageListener } = require('./lib/management/push'); const { saveConfigurationVersion } = require('./lib/management/configuration'); - // TODO: auth? // TODO: werelogs with a specific name. const CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS = 15000; - class ManagementAgentServer { constructor() { this.port = _config.managementAgent.port || 8010; @@ -34,9 +32,7 @@ class ManagementAgentServer { /* Define REPORT_TOKEN env variable needed by the management * module. */ - process.env.REPORT_TOKEN = process.env.REPORT_TOKEN - || _config.reportToken - || Uuid.v4(); + process.env.REPORT_TOKEN = process.env.REPORT_TOKEN || _config.reportToken || Uuid.v4(); initManagement(logger.newRequestLogger(), overlay => { let error = null; @@ -73,8 +69,7 @@ class ManagementAgentServer { this.wss.on('listening', this.onListening.bind(this)); this.wss.on('error', this.onError.bind(this)); - setInterval(this.checkBrokenConnections.bind(this), - CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS); + setInterval(this.checkBrokenConnections.bind(this), CHECK_BROKEN_CONNECTIONS_FREQUENCY_MS); addOverlayMessageListener(this.onNewOverlay.bind(this)); } @@ -114,8 +109,7 @@ class ManagementAgentServer { } onListening() { - logger.info('websocket server listening', - { port: this.port }); + logger.info('websocket server listening', { port: this.port }); } onError(error) { @@ -137,27 +131,24 @@ class ManagementAgentServer { }; client.send(JSON.stringify(msg), error => { if (error) { - logger.error( - 'failed to send remoteOverlay to management agent client', { - error, client: client._socket._peername, - }); + logger.error('failed to send remoteOverlay to management agent client', { + error, + client: client._socket._peername, + }); } }); } onNewOverlay(remoteOverlay) { const remoteOverlayObj = JSON.parse(remoteOverlay); - saveConfigurationVersion( - this.loadedOverlay, remoteOverlayObj, logger, err => { - if (err) { - logger.error('failed to save remote overlay', { err }); - return; - } - this.loadedOverlay = remoteOverlayObj; - this.wss.clients.forEach( - this._sendNewOverlayToClient.bind(this) - ); - }); + saveConfigurationVersion(this.loadedOverlay, remoteOverlayObj, logger, err => { + if (err) { + logger.error('failed to save remote overlay', { err }); + return; + } + this.loadedOverlay = remoteOverlayObj; + this.wss.clients.forEach(this._sendNewOverlayToClient.bind(this)); + }); } checkBrokenConnections() { diff --git a/mdserver.js b/mdserver.js index 4244be1cbc..c5113a46a3 100644 --- a/mdserver.js +++ b/mdserver.js @@ -1,8 +1,7 @@ 'use strict'; const { config } = require('./lib/Config.js'); -const MetadataFileServer = - require('arsenal').storage.metadata.file.MetadataFileServer; +const MetadataFileServer = require('arsenal').storage.metadata.file.MetadataFileServer; const logger = require('./lib/utilities/logger'); process.on('uncaughtException', err => { @@ -16,14 +15,15 @@ process.on('uncaughtException', err => { }); if (config.backends.metadata === 'file') { - const mdServer = new MetadataFileServer( - { bindAddress: config.metadataDaemon.bindAddress, - port: config.metadataDaemon.port, - path: config.metadataDaemon.metadataPath, - restEnabled: config.metadataDaemon.restEnabled, - restPort: config.metadataDaemon.restPort, - recordLog: config.recordLog, - versioning: { replicationGroupId: config.replicationGroupId }, - log: config.log }); + const mdServer = new MetadataFileServer({ + bindAddress: config.metadataDaemon.bindAddress, + port: config.metadataDaemon.port, + path: config.metadataDaemon.metadataPath, + restEnabled: config.metadataDaemon.restEnabled, + restPort: config.metadataDaemon.restPort, + recordLog: config.recordLog, + versioning: { replicationGroupId: config.replicationGroupId }, + log: config.log, + }); mdServer.startServer(); } diff --git a/monitoring/alerts.yaml b/monitoring/alerts.yaml index fc1621ba79..6c1f1b0a6e 100644 --- a/monitoring/alerts.yaml +++ b/monitoring/alerts.yaml @@ -34,149 +34,148 @@ x-inputs: value: 0.500 groups: -- name: CloudServer - rules: + - name: CloudServer + rules: + - alert: DataAccessS3EndpointDegraded + expr: sum(up{namespace="${namespace}", service="${service}"}) < ${replicas} + for: '30s' + labels: + severity: warning + annotations: + description: 'Less than 100% of S3 endpoints are up and healthy' + summary: 'Data Access service is degraded' - - alert: DataAccessS3EndpointDegraded - expr: sum(up{namespace="${namespace}", service="${service}"}) < ${replicas} - for: "30s" - labels: - severity: warning - annotations: - description: "Less than 100% of S3 endpoints are up and healthy" - summary: "Data Access service is degraded" + - alert: DataAccessS3EndpointCritical + expr: sum(up{namespace="${namespace}", service="${service}"}) * 2 < ${replicas} + for: '30s' + labels: + severity: critical + annotations: + description: 'Less than 50% of S3 endpoints are up and healthy' + summary: 'Data Access service is critical' - - alert: DataAccessS3EndpointCritical - expr: sum(up{namespace="${namespace}", service="${service}"}) * 2 < ${replicas} - for: "30s" - labels: - severity: critical - annotations: - description: "Less than 50% of S3 endpoints are up and healthy" - summary: "Data Access service is critical" + # As a platform admin I want to be alerted (warning) when the system errors are more than 3% of + # all the response codes + - alert: SystemErrorsWarning + expr: | + sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) + / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) + >= ${systemErrorsWarningThreshold} + for: 5m + labels: + severity: warning + annotations: + description: 'System errors represent more than 3% of all the response codes' + summary: 'High ratio of system erors' - # As a platform admin I want to be alerted (warning) when the system errors are more than 3% of - # all the response codes - - alert: SystemErrorsWarning - expr: | - sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) - / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) - >= ${systemErrorsWarningThreshold} - for: 5m - labels: - severity: warning - annotations: - description: "System errors represent more than 3% of all the response codes" - summary: "High ratio of system erors" + # As a platform admin I want to be alerted (critical) when the system errors are more than 5% of + # all the response codes + - alert: SystemErrorsCritical + expr: | + sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) + / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) + >= ${systemErrorsCriticalThreshold} + for: 5m + labels: + severity: critical + annotations: + description: 'System errors represent more than 5% of all the response codes' + summary: 'Very high ratio of system erors' - # As a platform admin I want to be alerted (critical) when the system errors are more than 5% of - # all the response codes - - alert: SystemErrorsCritical - expr: | - sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}", code=~"5.."}[1m])) - / sum(rate(s3_cloudserver_http_requests_total{namespace="${namespace}", service="${service}"}[1m])) - >= ${systemErrorsCriticalThreshold} - for: 5m - labels: - severity: critical - annotations: - description: "System errors represent more than 5% of all the response codes" - summary: "Very high ratio of system erors" + # As a platform admin I want to be alerted (warning) when a listing operation latency or a + # version listing operation latency is more than 300ms + - alert: ListingLatencyWarning + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + >= ${listingLatencyWarningThreshold} + for: 5m + labels: + severity: warning + annotations: + description: 'Latency of listing or version listing operations is more than 300ms' + summary: 'High listing latency' - # As a platform admin I want to be alerted (warning) when a listing operation latency or a - # version listing operation latency is more than 300ms - - alert: ListingLatencyWarning - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - >= ${listingLatencyWarningThreshold} - for: 5m - labels: - severity: warning - annotations: - description: "Latency of listing or version listing operations is more than 300ms" - summary: "High listing latency" + # As a platform admin I want to be alerted (critical) when a listing operation latency or a + # version listing operation latency is more than 500ms + - alert: ListingLatencyCritical + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) + >= ${listingLatencyCriticalThreshold} + for: 5m + labels: + severity: critical + annotations: + description: 'Latency of listing or version listing operations is more than 500ms' + summary: 'Very high listing latency' - # As a platform admin I want to be alerted (critical) when a listing operation latency or a - # version listing operation latency is more than 500ms - - alert: ListingLatencyCritical - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="listBucket"}[1m])) - >= ${listingLatencyCriticalThreshold} - for: 5m - labels: - severity: critical - annotations: - description: "Latency of listing or version listing operations is more than 500ms" - summary: "Very high listing latency" + # As a platform admin I want to be alerted (warning) when a delete operation latency is more than + # 500ms + - alert: DeleteLatencyWarning + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + >= ${deleteLatencyWarningThreshold} + for: 5m + labels: + severity: warning + annotations: + description: 'Latency of delete object operations is more than 500ms' + summary: 'High delete latency' - # As a platform admin I want to be alerted (warning) when a delete operation latency is more than - # 500ms - - alert: DeleteLatencyWarning - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - >= ${deleteLatencyWarningThreshold} - for: 5m - labels: - severity: warning - annotations: - description: "Latency of delete object operations is more than 500ms" - summary: "High delete latency" + # As a platform admin I want to be alerted (critical) when a delete operation latency is more + # than 1s + - alert: DeleteLatencyCritical + expr: | + sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) + >= ${deleteLatencyCriticalThreshold} + for: 5m + labels: + severity: critical + annotations: + description: 'Latency of delete object operations is more than 1s' + summary: 'Very high delete latency' - # As a platform admin I want to be alerted (critical) when a delete operation latency is more - # than 1s - - alert: DeleteLatencyCritical - expr: | - sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - / sum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace="${namespace}",service="${service}",action="deleteObject"}[1m])) - >= ${deleteLatencyCriticalThreshold} - for: 5m - labels: - severity: critical - annotations: - description: "Latency of delete object operations is more than 1s" - summary: "Very high delete latency" + # As a platform admin I want to be alerted (warning) when the utilization metrics service is enabled + # but not available for at least half of the S3 services during the last minute + - alert: QuotaMetricsNotAvailable + expr: | + avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) + < ${quotaUnavailabilityThreshold} and + (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or + max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) + labels: + severity: warning + annotations: + description: 'The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.' + summary: 'Utilization metrics service not available' - # As a platform admin I want to be alerted (warning) when the utilization metrics service is enabled - # but not available for at least half of the S3 services during the last minute - - alert: QuotaMetricsNotAvailable - expr: | - avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) - < ${quotaUnavailabilityThreshold} and - (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or - max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) - labels: - severity: warning - annotations: - description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled." - summary: "Utilization metrics service not available" + # As a platform admin I want to be alerted (critical) when the utilization metrics service is enabled + # but not available during the last 10 minutes + - alert: QuotaMetricsNotAvailable + expr: | + avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) + < ${quotaUnavailabilityThreshold} and + (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or + max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) + for: 10m + labels: + severity: critical + annotations: + description: 'The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled.' + summary: 'Utilization metrics service not available' - # As a platform admin I want to be alerted (critical) when the utilization metrics service is enabled - # but not available during the last 10 minutes - - alert: QuotaMetricsNotAvailable - expr: | - avg(s3_cloudserver_quota_utilization_service_available{namespace="${namespace}",service="${service}"}) - < ${quotaUnavailabilityThreshold} and - (max(s3_cloudserver_quota_buckets_count{namespace="${namespace}", job="${reportJob}"}) > 0 or - max(s3_cloudserver_quota_accounts_count{namespace="${namespace}", job="${reportJob}"}) > 0) - for: 10m - labels: - severity: critical - annotations: - description: "The storage metrics required for Account or S3 Bucket Quota checks are not available, the quotas are disabled." - summary: "Utilization metrics service not available" - - # As a platform admin I want to be alerted (critical) when quotas were not honored due to metrics - # being unavailable - - alert: QuotaUnavailable - expr: | - sum(increase(s3_cloudserver_quota_unavailable_count{namespace="${namespace}",service="${service}"}[2m])) - > 0 - for: 5m - labels: - severity: critical - annotations: - description: "Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet." - summary: "High number of quota requests with metrics unavailable" + # As a platform admin I want to be alerted (critical) when quotas were not honored due to metrics + # being unavailable + - alert: QuotaUnavailable + expr: | + sum(increase(s3_cloudserver_quota_unavailable_count{namespace="${namespace}",service="${service}"}[2m])) + > 0 + for: 5m + labels: + severity: critical + annotations: + description: 'Quotas were not honored due to metrics being unavailable. If the S3 Bucket or Account was created recently, the metrics may not be available yet.' + summary: 'High number of quota requests with metrics unavailable' diff --git a/monitoring/dashboard.json b/monitoring/dashboard.json index 1eca3701c1..7b6664a4b0 100644 --- a/monitoring/dashboard.json +++ b/monitoring/dashboard.json @@ -1,3629 +1,3528 @@ { - "__inputs": [ - { - "description": "", - "label": "Prometheus", - "name": "DS_PROMETHEUS", - "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" - }, - { - "description": "", - "label": "Loki", - "name": "DS_LOKI", - "pluginId": "loki", - "pluginName": "Loki", - "type": "datasource" - }, - { - "description": "Namespace associated with the Zenko instance", - "label": "namespace", - "name": "namespace", - "type": "constant", - "value": "zenko" - }, - { - "description": "Name of the Zenko instance", - "label": "instance", - "name": "zenkoName", - "type": "constant", - "value": "artesca-data" - }, - { - "description": "Name of the Cloudserver container, used to filter only the Cloudserver services.", - "label": "container", - "name": "container", - "type": "constant", - "value": "connector-cloudserver" - }, - { - "description": "Name of the Cloudserver Report job, used to filter only the Report Handler instances.", - "label": "report job", - "name": "reportJob", - "type": "constant", - "value": "artesca-data-ops-report-handler" - }, - { - "description": "Name of the Count-Items cronjob, used to filter only the Count-Items instances.", - "label": "count-items job", - "name": "countItemsJob", - "type": "constant", - "value": "artesca-data-ops-count-items" - } - ], - "annotations": { - "list": [] - }, - "description": "", - "editable": true, - "gnetId": null, - "hideControls": false, - "id": null, - "links": [], - "panels": [ - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 1.0, - "yaxis": "left" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 0 - }, - "hideTimeOverride": false, - "id": 1, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ + "__inputs": [ { - "datasource": null, - "expr": "sum(up{namespace=\"${namespace}\", job=~\"$job\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Up", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" + "description": "", + "label": "Prometheus", + "name": "DS_PROMETHEUS", + "pluginId": "prometheus", + "pluginName": "Prometheus", + "type": "datasource" }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 3, - "y": 0 - }, - "hideTimeOverride": false, - "id": 2, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Http requests rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "calcs": [ - "mean" - ], - "decimals": null, - "limit": null, - "links": [], - "mappings": [], - "max": 100, - "min": 0, - "noValue": "-", - "override": {}, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "red", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - }, - { - "color": "orange", - "index": 2, - "line": true, - "op": "gt", - "value": 80.0, - "yaxis": "left" - }, - { - "color": "green", - "index": 3, - "line": true, - "op": "gt", - "value": 90.0, - "yaxis": "left" - } - ] - }, - "title": null, - "unit": "percent", - "values": false + "description": "", + "label": "Loki", + "name": "DS_LOKI", + "pluginId": "loki", + "pluginName": "Loki", + "type": "datasource" }, - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 6, - "y": 0 - }, - "hideTimeOverride": false, - "id": 3, - "links": [], - "maxDataPoints": 100, - "options": { - "reduceOptions": { - "calcs": [ - "mean" - ] - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval])) * 100\n /\nsum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]) > 0)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Success rate", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Success rate", - "transformations": [], - "transparent": false, - "type": "gauge" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of data ingested : cumulative amount of data created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 0 - }, - "hideTimeOverride": false, - "id": 4, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "description": "Namespace associated with the Zenko instance", + "label": "namespace", + "name": "namespace", + "type": "constant", + "value": "zenko" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "-sum(deriv(s3_cloudserver_disk_available_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Injection Data Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of object ingestion : cumulative count of object created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "dark-purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "O/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 0 - }, - "hideTimeOverride": false, - "id": 5, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "description": "Name of the Zenko instance", + "label": "instance", + "name": "zenkoName", + "type": "constant", + "value": "artesca-data" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(deriv(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Injection Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of S3 buckets available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 7, - "x": 15, - "y": 0 - }, - "hideTimeOverride": false, - "id": 6, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "description": "Name of the Cloudserver container, used to filter only the Cloudserver services.", + "label": "container", + "name": "container", + "type": "constant", + "value": "connector-cloudserver" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(s3_cloudserver_buckets_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Buckets", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Status of the reports-handler pod.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "red", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 1.0, - "yaxis": "left" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 0 - }, - "hideTimeOverride": false, - "id": 7, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "description": "Name of the Cloudserver Report job, used to filter only the Report Handler instances.", + "label": "report job", + "name": "reportJob", + "type": "constant", + "value": "artesca-data-ops-report-handler" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(up{namespace=\"${namespace}\", job=\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "description": "Name of the Count-Items cronjob, used to filter only the Count-Items instances.", + "label": "count-items job", + "name": "countItemsJob", + "type": "constant", + "value": "artesca-data-ops-count-items" } - ], - "title": "Reporter", - "transformations": [], - "transparent": false, - "type": "stat" + ], + "annotations": { + "list": [] }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 0, - "y": 4 - }, - "hideTimeOverride": false, - "id": 8, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "targets": [ + "description": "", + "editable": true, + "gnetId": null, + "hideControls": false, + "id": null, + "links": [], + "panels": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=\"200\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Status 200", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 3, - "y": 4 - }, - "hideTimeOverride": false, - "id": 9, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 1.0, + "yaxis": "left" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 0 + }, + "hideTimeOverride": false, + "id": 1, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(up{namespace=\"${namespace}\", job=~\"$job\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Up", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"4..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Status 4xx", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "0", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "semi-dark-blue", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 5, - "y": 4 - }, - "hideTimeOverride": false, - "id": 10, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 0 + }, + "hideTimeOverride": false, + "id": 2, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Http requests rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"5..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Status 5xx", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "red", - "index": 1, - "line": true, - "op": "gt", - "value": 80.0, - "yaxis": "left" - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 7, - "y": 4 - }, - "hideTimeOverride": false, - "id": 11, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "calcs": ["mean"], + "decimals": null, + "limit": null, + "links": [], + "mappings": [], + "max": 100, + "min": 0, + "noValue": "-", + "override": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + }, + { + "color": "orange", + "index": 2, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + }, + { + "color": "green", + "index": 3, + "line": true, + "op": "gt", + "value": 90.0, + "yaxis": "left" + } + ] + }, + "title": null, + "unit": "percent", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 0 + }, + "hideTimeOverride": false, + "id": 3, + "links": [], + "maxDataPoints": 100, + "options": { + "reduceOptions": { + "calcs": ["mean"] + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval])) * 100\n /\nsum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]) > 0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Success rate", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Success rate", + "transformations": [], + "transparent": false, + "type": "gauge" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", job=~\"$job\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Active requests", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of data ingested out-of-band (OOB) : cumulative amount of OOB data created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "binBps" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 9, - "y": 4 - }, - "hideTimeOverride": false, - "id": 12, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of data ingested : cumulative amount of data created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 0 + }, + "hideTimeOverride": false, + "id": 4, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "-sum(deriv(s3_cloudserver_disk_available_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Injection Data Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(deriv(s3_cloudserver_ingested_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "OOB Inject. Data Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Rate of object ingested out-of-band (OOB) : cumulative count of OOB object created (>0) or deleted (<0) per second.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": 1, - "mappings": [], - "noValue": "none", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "purple", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - } - ] - }, - "unit": "O/s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 12, - "y": 4 - }, - "hideTimeOverride": false, - "id": 13, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "background", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of object ingestion : cumulative count of object created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "O/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 0 + }, + "hideTimeOverride": false, + "id": 5, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(deriv(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Injection Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(deriv(s3_cloudserver_ingested_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "OOB Inject. Rate", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of S3 objects available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 7, - "x": 15, - "y": 4 - }, - "hideTimeOverride": false, - "id": 14, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Number of S3 buckets available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 7, + "x": 15, + "y": 0 + }, + "hideTimeOverride": false, + "id": 6, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(s3_cloudserver_buckets_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Buckets", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "sum(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Objects", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Time elapsed since the last report, when object/bucket count was updated.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - }, - { - "color": "super-light-yellow", - "index": 2, - "line": true, - "op": "gt", - "value": 1800.0, - "yaxis": "left" - }, - { - "color": "orange", - "index": 3, - "line": true, - "op": "gt", - "value": 3600.0, - "yaxis": "left" - }, - { - "color": "red", - "index": 4, - "line": true, - "op": "gt", - "value": 3700.0, - "yaxis": "left" - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 2, - "x": 22, - "y": 4 - }, - "hideTimeOverride": false, - "id": 15, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Status of the reports-handler pod.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 1.0, + "yaxis": "left" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 0 + }, + "hideTimeOverride": false, + "id": 7, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(up{namespace=\"${namespace}\", job=\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Reporter", + "transformations": [], + "transparent": false, + "type": "stat" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "time()\n- max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n+ (max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n - max(kube_cronjob_status_last_schedule_time{namespace=\"${namespace}\", cronjob=\"${countItemsJob}\"})\n > 0 or vector(0))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Last Report", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 8 - }, - "hideTimeOverride": false, - "id": 16, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Response codes", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "hideTimeOverride": false, - "id": 17, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 4 + }, + "hideTimeOverride": false, + "id": 8, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=\"200\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Status 200", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum by (code) (rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{code}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Http status code over time", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 39, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Success" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "User errors" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-orange", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "System errors" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "semi-dark-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "hideTimeOverride": false, - "id": 18, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 3, + "y": 4 + }, + "hideTimeOverride": false, + "id": 9, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Status 4xx", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Success", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-blue", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 5, + "y": 4 + }, + "hideTimeOverride": false, + "id": 10, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\",code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Status 5xx", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"4..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "User errors", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 7, + "y": 4 + }, + "hideTimeOverride": false, + "id": 11, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", job=~\"$job\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Active requests", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"5..\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "System errors", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Aggregated status over time", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 17 - }, - "hideTimeOverride": false, - "id": 19, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Operations", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 18, - "x": 0, - "y": 18 - }, - "hideTimeOverride": false, - "id": 20, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "right" + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of data ingested out-of-band (OOB) : cumulative amount of OOB data created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 9, + "y": 4 + }, + "hideTimeOverride": false, + "id": 12, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(deriv(s3_cloudserver_ingested_bytes{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "OOB Inject. Data Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by(action)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{action}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Request rate per S3 action", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 6, - "x": 18, - "y": 18 - }, - "hideTimeOverride": false, - "id": 21, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name", - "percent" - ], - "legend": { - "displayMode": "list", - "placement": "bottom", - "values": [] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "sum" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "description": "Rate of object ingested out-of-band (OOB) : cumulative count of OOB object created (>0) or deleted (<0) per second.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "noValue": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "purple", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + } + ] + }, + "unit": "O/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 12, + "y": 4 + }, + "hideTimeOverride": false, + "id": 13, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(deriv(s3_cloudserver_ingested_objects_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "OOB Inject. Rate", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(round(increase(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))) by(method)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{method}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "HTTP Method breakdown", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 28 - }, - "hideTimeOverride": false, - "id": 22, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Latency", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 29 - }, - "hideTimeOverride": false, - "id": 23, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + "datasource": "${DS_PROMETHEUS}", + "description": "Number of S3 objects available in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 7, + "x": 15, + "y": 4 + }, + "hideTimeOverride": false, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "sum(s3_cloudserver_objects_count{namespace=\"${namespace}\", job=\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Objects", + "transformations": [], + "transparent": false, + "type": "stat" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Overall", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "description": "Time elapsed since the last report, when object/bucket count was updated.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + }, + { + "color": "super-light-yellow", + "index": 2, + "line": true, + "op": "gt", + "value": 1800.0, + "yaxis": "left" + }, + { + "color": "orange", + "index": 3, + "line": true, + "op": "gt", + "value": 3600.0, + "yaxis": "left" + }, + { + "color": "red", + "index": 4, + "line": true, + "op": "gt", + "value": 3700.0, + "yaxis": "left" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 4 + }, + "hideTimeOverride": false, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "time()\n- max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n+ (max(s3_cloudserver_last_report_timestamp{namespace=\"${namespace}\", job=\"${reportJob}\"})\n - max(kube_cronjob_status_last_schedule_time{namespace=\"${namespace}\", cronjob=\"${countItemsJob}\"})\n > 0 or vector(0))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Last Report", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Upload", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Response codes", + "transformations": [], + "transparent": false, + "type": "row" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Delete", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "hideTimeOverride": false, + "id": 17, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum by (code) (rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{code}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Http status code over time", + "transformations": [], + "transparent": false, + "type": "timeseries" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Download", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 39, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "User errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "System errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "semi-dark-red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "hideTimeOverride": false, + "id": 18, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"2..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Success", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"4..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "User errors", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\", code=~\"5..\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "System errors", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Aggregated status over time", + "transformations": [], + "transparent": false, + "type": "timeseries" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Multi-delete", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Average latencies", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 29 - }, - "heatmap": {}, - "hideTimeOverride": false, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 24, - "legend": { - "show": false - }, - "links": [], - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ - { - "datasource": null, - "expr": "sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Request duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "transformations": [], - "transparent": false, - "type": "heatmap", - "xAxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yAxis": { - "decimals": null, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 10, - "w": 24, - "x": 0, - "y": 37 - }, - "hideTimeOverride": false, - "id": 25, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "max", - "mean" - ], - "displayMode": "table", - "placement": "right" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "hideTimeOverride": false, + "id": 19, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Operations", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{action}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Latencies per S3 action", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 47 - }, - "hideTimeOverride": false, - "id": 26, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Data rate", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "red", - "index": 1, - "line": true, - "op": "gt", - "value": 80.0, - "yaxis": "left" - } - ] - }, - "unit": "binBps" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 18, + "x": 0, + "y": 18 + }, + "hideTimeOverride": false, + "id": 20, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["min", "mean", "max"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by(action)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Request rate per S3 action", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Out" - }, - "properties": [ - { - "id": "custom.axisPlacement", - "value": "right" - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "hideTimeOverride": false, - "id": 27, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 18 + }, + "hideTimeOverride": false, + "id": 21, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name", "percent"], + "legend": { + "displayMode": "list", + "placement": "bottom", + "values": [] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": ["sum"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(round(increase(s3_cloudserver_http_requests_total{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))) by(method)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{method}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "HTTP Method breakdown", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Out", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "hideTimeOverride": false, + "id": 22, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Latency", + "transformations": [], + "transparent": false, + "type": "row" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "In", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Bandwidth", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - } - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 12, - "y": 48 - }, - "hideTimeOverride": false, - "id": 28, - "links": [], - "maxDataPoints": 100, - "options": { - "displayMode": "gradient", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "decimals": null, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "hideTimeOverride": false, + "id": 23, "links": [], - "max": null, - "min": null, - "noValue": "-", - "title": null, - "unit": "bytes" - }, - "limit": null, - "mappings": [], - "override": {}, - "thresholds": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ], - "values": false + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Overall", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"objectPut|objectPutPart|objectCopy|objectPutCopyPart\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Upload", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectDelete\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Delete", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=\"objectGet\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Download", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\", action=~\"multiObjectDelete|multipartDelete\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Multi-delete", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Average latencies", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "orientation": "vertical", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ { - "datasource": null, - "expr": "avg(s3_cloudserver_http_request_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ quantile }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Avg upload chunk size by \u03c6-quantile", - "transformations": [], - "transparent": false, - "type": "bargauge" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - } - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 48 - }, - "hideTimeOverride": false, - "id": 29, - "links": [], - "maxDataPoints": 100, - "options": { - "displayMode": "gradient", - "fieldOptions": { - "calcs": [ - "lastNotNull" - ], - "defaults": { - "decimals": null, + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 29 + }, + "heatmap": {}, + "hideTimeOverride": false, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 24, + "legend": { + "show": false + }, "links": [], - "max": null, - "min": null, - "noValue": "-", - "title": null, - "unit": "bytes" - }, - "limit": null, - "mappings": [], - "override": {}, - "thresholds": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (increase(s3_cloudserver_http_request_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Request duration", + "tooltip": { + "show": true, + "showHistogram": true }, - { - "color": "green", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yAxis": { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true } - ], - "values": false }, - "orientation": "vertical", - "showThresholdLabels": false, - "showThresholdMarkers": true - }, - "targets": [ { - "datasource": null, - "expr": "avg(s3_cloudserver_http_response_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ quantile }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Avg download chunk size by \u03c6-quantile", - "transformations": [], - "transparent": false, - "type": "bargauge" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 56 - }, - "hideTimeOverride": false, - "id": 30, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Errors", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_LOKI}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 57 - }, - "hideTimeOverride": false, - "id": 31, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 37 + }, + "hideTimeOverride": false, + "id": 25, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["max", "mean"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_duration_seconds_sum{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)\n /\nsum(rate(s3_cloudserver_http_request_duration_seconds_count{namespace=\"${namespace}\", job=~\"$job\"}[$__rate_interval])) by (action)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Latencies per S3 action", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"404\"\n [$__interval])\n))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{bucketName}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "404 : Top10 by Bucket", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "datasource": "${DS_LOKI}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 57 - }, - "hideTimeOverride": false, - "id": 32, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 47 + }, + "hideTimeOverride": false, + "id": 26, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Data rate", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"500\"\n [$__interval])\n))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{bucketName}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "500 : Top10 by Bucket", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "datasource": "${DS_LOKI}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 57 - }, - "hideTimeOverride": false, - "id": 33, - "links": [], - "maxDataPoints": 100, - "options": { - "displayLabels": [ - "name" - ], - "legend": { - "displayMode": "table", - "placement": "right", - "values": [ - "value" - ] - }, - "pieType": "donut", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "red", + "index": 1, + "line": true, + "op": "gt", + "value": 80.0, + "yaxis": "left" + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Out" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 48 + }, + "hideTimeOverride": false, + "id": 27, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_response_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Out", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_http_request_size_bytes_sum{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "In", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Bandwidth", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=~\"5..\"\n [$__interval])\n))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{bucketName}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "5xx : Top10 by Bucket", - "transformations": [], - "transparent": false, - "type": "piechart" - }, - { - "collapsed": false, - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 65 - }, - "hideTimeOverride": false, - "id": 34, - "links": [], - "maxDataPoints": 100, - "panels": [], - "targets": [], - "title": "Quotas", - "transformations": [], - "transparent": false, - "type": "row" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of S3 buckets with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 66 - }, - "hideTimeOverride": false, - "id": 35, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 48 + }, + "hideTimeOverride": false, + "id": 28, + "links": [], + "maxDataPoints": 100, + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": ["lastNotNull"], + "defaults": { + "decimals": null, + "links": [], + "max": null, + "min": null, + "noValue": "-", + "title": null, + "unit": "bytes" + }, + "limit": null, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ], + "values": false + }, + "orientation": "vertical", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": null, + "expr": "avg(s3_cloudserver_http_request_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ quantile }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Avg upload chunk size by \u03c6-quantile", + "transformations": [], + "transparent": false, + "type": "bargauge" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "max(s3_cloudserver_quota_buckets_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Buckets with quota", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "description": "Number of accounts with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "custom": {}, - "decimals": null, - "mappings": [], - "noValue": "-", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "#808080", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "blue", - "index": 1, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 6, - "x": 0, - "y": 70 - }, - "hideTimeOverride": false, - "id": 36, - "links": [], - "maxDataPoints": 100, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 48 + }, + "hideTimeOverride": false, + "id": 29, + "links": [], + "maxDataPoints": 100, + "options": { + "displayMode": "gradient", + "fieldOptions": { + "calcs": ["lastNotNull"], + "defaults": { + "decimals": null, + "links": [], + "max": null, + "min": null, + "noValue": "-", + "title": null, + "unit": "bytes" + }, + "limit": null, + "mappings": [], + "override": {}, + "thresholds": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "green", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ], + "values": false + }, + "orientation": "vertical", + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "targets": [ + { + "datasource": null, + "expr": "avg(s3_cloudserver_http_response_size_bytes{namespace=\"${namespace}\", job=~\"$job\"}) by (quantile)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ quantile }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Avg download chunk size by \u03c6-quantile", + "transformations": [], + "transparent": false, + "type": "bargauge" }, - "textMode": "auto" - }, - "targets": [ { - "datasource": null, - "expr": "max(s3_cloudserver_quota_accounts_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Accounts with quota", - "transformations": [], - "transparent": false, - "type": "stat" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 66 - }, - "hideTimeOverride": false, - "id": 37, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 56 + }, + "hideTimeOverride": false, + "id": 30, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Errors", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_unavailable_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Operations with unavailable metrics", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 66 - }, - "hideTimeOverride": false, - "id": 38, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "right" + "datasource": "${DS_LOKI}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 57 + }, + "hideTimeOverride": false, + "id": 31, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"404\"\n [$__interval])\n))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucketName}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "404 : Top10 by Bucket", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval])) by(action)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{action}}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota evaluaton rate per S3 action", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "stepAfter", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "index": 0, - "line": true, - "op": "gt", - "value": "null", - "yaxis": "left" - }, - { - "color": "orange", - "index": 1, - "line": true, - "op": "gt", - "value": 90.0, - "yaxis": "left" - }, - { - "color": "red", - "index": 2, - "line": true, - "op": "gt", - "value": 0.0, - "yaxis": "left" - } - ] - }, - "unit": "percent" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 74 - }, - "hideTimeOverride": false, - "id": 39, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "hidden", - "placement": "bottom" + "datasource": "${DS_LOKI}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 57 + }, + "hideTimeOverride": false, + "id": 32, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=\"500\"\n [$__interval])\n))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucketName}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "500 : Top10 by Bucket", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "avg(avg_over_time(s3_cloudserver_quota_utilization_service_available{namespace=\"${namespace}\",job=\"${job}\"}[$__rate_interval])) * 100", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota service uptime", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 30, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "ops" + "datasource": "${DS_LOKI}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 57 + }, + "hideTimeOverride": false, + "id": 33, + "links": [], + "maxDataPoints": 100, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "table", + "placement": "right", + "values": ["value"] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "topk(10, sum by(bucketName) (\n count_over_time({namespace=\"${namespace}\", pod=~\"$pod\"}\n | json | bucketName!=\"\" and httpCode=~\"5..\"\n [$__interval])\n))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucketName}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "5xx : Top10 by Bucket", + "transformations": [], + "transparent": false, + "type": "piechart" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 6, - "y": 74 - }, - "hideTimeOverride": false, - "id": 40, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "collapsed": false, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "hideTimeOverride": false, + "id": 34, + "links": [], + "maxDataPoints": 100, + "panels": [], + "targets": [], + "title": "Quotas", + "transformations": [], + "transparent": false, + "type": "row" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=~\"2..\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Success", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "description": "Number of S3 buckets with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 66 + }, + "hideTimeOverride": false, + "id": 35, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "max(s3_cloudserver_quota_buckets_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Buckets with quota", + "transformations": [], + "transparent": false, + "type": "stat" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=\"429\", job=\"${job}\"}[$__rate_interval]))", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "Quota Exceeded", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota evaluation status code over time", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" + "datasource": "${DS_PROMETHEUS}", + "description": "Number of accounts with quota enabled in the cluster.\nThis value is computed asynchronously, and update may be delayed up to 1h.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": null, + "mappings": [], + "noValue": "-", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#808080", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "blue", + "index": 1, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 70 + }, + "hideTimeOverride": false, + "id": 36, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "targets": [ + { + "datasource": null, + "expr": "max(s3_cloudserver_quota_accounts_count{namespace=\"${namespace}\", job=~\"${reportJob}\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Accounts with quota", + "transformations": [], + "transparent": false, + "type": "stat" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 74 - }, - "hideTimeOverride": false, - "id": 41, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [ - "min", - "mean", - "max" - ], - "displayMode": "table", - "placement": "right" + { + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 66 + }, + "hideTimeOverride": false, + "id": 37, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_unavailable_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Operations with unavailable metrics", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ type }} (success)", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 66 + }, + "hideTimeOverride": false, + "id": 38, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["min", "mean", "max"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval])) by(action)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{action}}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota evaluaton rate per S3 action", + "transformations": [], + "transparent": false, + "type": "timeseries" }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ type }} (exceeded)", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Average quota evaluation latencies", - "transformations": [], - "transparent": false, - "type": "timeseries" - }, - { - "cards": { - "cardPadding": null, - "cardRound": null - }, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateOranges", - "exponent": 0.5, - "max": null, - "min": null, - "mode": "opacity" - }, - "dataFormat": "tsbuckets", - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "thresholds": { - "mode": "absolute", - "steps": [] - } - } - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 82 - }, - "heatmap": {}, - "hideTimeOverride": false, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 42, - "legend": { - "show": false - }, - "links": [], - "maxDataPoints": 25, - "reverseYBuckets": false, - "targets": [ + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "stepAfter", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "index": 0, + "line": true, + "op": "gt", + "value": "null", + "yaxis": "left" + }, + { + "color": "orange", + "index": 1, + "line": true, + "op": "gt", + "value": 90.0, + "yaxis": "left" + }, + { + "color": "red", + "index": 2, + "line": true, + "op": "gt", + "value": 0.0, + "yaxis": "left" + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 74 + }, + "hideTimeOverride": false, + "id": 39, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "avg(avg_over_time(s3_cloudserver_quota_utilization_service_available{namespace=\"${namespace}\",job=\"${job}\"}[$__rate_interval])) * 100", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota service uptime", + "transformations": [], + "transparent": false, + "type": "timeseries" + }, { - "datasource": null, - "expr": "sum by(le) (increase(s3_cloudserver_quota_evaluation_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", - "format": "heatmap", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ le }}", - "metric": "", - "refId": "", - "step": 10, - "target": "" - } - ], - "title": "Quota evaluation duration", - "tooltip": { - "show": true, - "showHistogram": true - }, - "transformations": [], - "transparent": false, - "type": "heatmap", - "xAxis": { - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yAxis": { - "decimals": null, - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - }, - { - "datasource": "${DS_PROMETHEUS}", - "editable": true, - "error": false, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "smooth", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": 180000, - "stacking": {}, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [] - }, - "unit": "s" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 74 + }, + "hideTimeOverride": false, + "id": 40, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=~\"2..\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Success", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", code=\"429\", job=\"${job}\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Quota Exceeded", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota evaluation status code over time", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 18, - "x": 6, - "y": 82 - }, - "hideTimeOverride": false, - "id": 43, - "links": [], - "maxDataPoints": 100, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom" + { + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 74 + }, + "hideTimeOverride": false, + "id": 41, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": ["min", "mean", "max"], + "displayMode": "table", + "placement": "right" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (type)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} (success)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_evaluation_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)\n /\nsum(rate(s3_cloudserver_quota_evaluation_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=\"429\"}[$__rate_interval])) by (type)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ type }} (exceeded)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Average quota evaluation latencies", + "transformations": [], + "transparent": false, + "type": "timeseries" }, - "tooltip": { - "mode": "single" - } - }, - "targets": [ { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ class }} (success)", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateOranges", + "exponent": 0.5, + "max": null, + "min": null, + "mode": "opacity" + }, + "dataFormat": "tsbuckets", + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "thresholds": { + "mode": "absolute", + "steps": [] + } + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 82 + }, + "heatmap": {}, + "hideTimeOverride": false, + "hideZeroBuckets": false, + "highlightCards": true, + "id": 42, + "legend": { + "show": false + }, + "links": [], + "maxDataPoints": 25, + "reverseYBuckets": false, + "targets": [ + { + "datasource": null, + "expr": "sum by(le) (increase(s3_cloudserver_quota_evaluation_duration_seconds_bucket{namespace=\"${namespace}\", job=\"${job}\"}[$__rate_interval]))", + "format": "heatmap", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ le }}", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Quota evaluation duration", + "tooltip": { + "show": true, + "showHistogram": true + }, + "transformations": [], + "transparent": false, + "type": "heatmap", + "xAxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yAxis": { + "decimals": null, + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } }, { - "datasource": null, - "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)", - "format": "time_series", - "hide": false, - "instant": false, - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{ class }} (error)", - "metric": "", - "refId": "", - "step": 10, - "target": "" + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 180000, + "stacking": {}, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 18, + "x": 6, + "y": 82 + }, + "hideTimeOverride": false, + "id": 43, + "links": [], + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "targets": [ + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"2..\"}[$__rate_interval])) by (class)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ class }} (success)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + }, + { + "datasource": null, + "expr": "sum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_sum{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)\n /\nsum(rate(s3_cloudserver_quota_metrics_retrieval_duration_seconds_count{namespace=\"${namespace}\", job=\"${job}\", code=~\"4..|5..\"}[$__rate_interval])) by (class)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ class }} (error)", + "metric": "", + "refId": "", + "step": 10, + "target": "" + } + ], + "title": "Average utilization metrics retrieval latencies", + "transformations": [], + "transparent": false, + "type": "timeseries" } - ], - "title": "Average utilization metrics retrieval latencies", - "transformations": [], - "transparent": false, - "type": "timeseries" - } - ], - "refresh": "30s", - "rows": [], - "schemaVersion": 12, - "sharedCrosshair": false, - "style": "dark", - "tags": [ - "CloudServer" - ], - "templating": { - "list": [ - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "tags": [], - "text": null, - "value": null - }, - "datasource": "${DS_PROMETHEUS}", - "hide": 0, - "includeAll": false, - "label": "Group", - "multi": true, - "name": "job", - "options": [], - "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\"}, job)", - "refresh": 1, - "regex": "/(?${zenkoName}-(?\\w*).*)/", - "sort": 1, - "tagValuesQuery": null, - "tagsQuery": null, - "type": "query", - "useTags": false - }, - { - "allValue": null, - "auto": false, - "auto_count": 30, - "auto_min": "10s", - "current": { - "selected": false, - "tags": [], - "text": null, - "value": null - }, - "datasource": "${DS_PROMETHEUS}", - "hide": 2, - "includeAll": false, - "label": "pod", - "multi": false, - "name": "pod", - "options": [], - "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\", job=~\"$job\"}, pod)", - "refresh": 1, - "regex": null, - "sort": 1, - "tagValuesQuery": null, - "tagsQuery": null, - "type": "query", - "useTags": false - } - ] - }, - "time": { - "from": "now-1h", - "to": "now" - }, - "timepicker": { - "hidden": false, - "refresh_intervals": [ - "5s", - "10s", - "30s", - "1m", - "5m", - "15m", - "30m", - "1h", - "2h", - "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] - }, - "timezone": "", - "title": "S3 service", - "uid": null, - "version": 110 + "refresh": "30s", + "rows": [], + "schemaVersion": 12, + "sharedCrosshair": false, + "style": "dark", + "tags": ["CloudServer"], + "templating": { + "list": [ + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": false, + "label": "Group", + "multi": true, + "name": "job", + "options": [], + "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\"}, job)", + "refresh": 1, + "regex": "/(?${zenkoName}-(?\\w*).*)/", + "sort": 1, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "tags": [], + "text": null, + "value": null + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 2, + "includeAll": false, + "label": "pod", + "multi": false, + "name": "pod", + "options": [], + "query": "label_values(s3_cloudserver_http_active_requests{namespace=\"${namespace}\", container=\"${container}\", job=~\"$job\"}, pod)", + "refresh": 1, + "regex": null, + "sort": 1, + "tagValuesQuery": null, + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": false, + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "", + "title": "S3 service", + "uid": null, + "version": 110 } diff --git a/package.json b/package.json index 5fb4b68aea..f6ce19e58e 100644 --- a/package.json +++ b/package.json @@ -145,6 +145,9 @@ "ft_sse_arn": "cd tests/functional/sse-kms-migration && mocha --reporter mocha-multi-reporters --reporter-options configFile=$INIT_CWD/tests/reporter-config.json -t 10000 cleanup.js arnPrefix.js --exit", "lint": "eslint $(git ls-files '*.js')", "lint_md": "mdlint $(git ls-files '*.md')", + "prettier": "prettier", + "prettier:check": "prettier --check .", + "prettier:write": "prettier --write .", "mem_backend": "S3BACKEND=mem node index.js", "start": "npm-run-all --parallel start_dmd start_s3server", "start_mongo": "yarn run cloudserver", @@ -168,8 +171,6 @@ "cover": "nyc --clean --silent yarn run", "postcover": "nyc report --report-dir ./coverage/test --reporter=lcov", "count-async": "node .github/scripts/count-async-functions.mjs", - "check-diff-async": "node .github/scripts/check-diff-async.mjs", - "prettier": "prettier", - "prettier:diff": "bash scripts/prettier-diff.sh" + "check-diff-async": "node .github/scripts/check-diff-async.mjs" } } diff --git a/scripts/prettier-diff.sh b/scripts/prettier-diff.sh deleted file mode 100644 index 90db614e4a..0000000000 --- a/scripts/prettier-diff.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Purpose: -# - Run Prettier on files returned by `git diff --name-only`. -# - Keep CI focused on changed files while remaining flexible. -# -# Usage examples: -# - scripts/prettier-diff.sh -# - scripts/prettier-diff.sh --check -# - scripts/prettier-diff.sh --format -# - scripts/prettier-diff.sh --check HEAD~1..HEAD -# - scripts/prettier-diff.sh --write --cached - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT_DIR" - -PRETTIER_ARGS=() -HAS_MODE=false -DIFF_ARGS=() -for arg in "$@"; do - case "$arg" in - --format) - PRETTIER_ARGS+=("--write") - HAS_MODE=true - ;; - --check|--write) - PRETTIER_ARGS+=("$arg") - HAS_MODE=true - ;; - *) - DIFF_ARGS+=("$arg") - ;; - esac -done - -if [[ "$HAS_MODE" = false ]]; then - PRETTIER_ARGS=("--check" "${PRETTIER_ARGS[@]}") -fi - -mapfile -t CHANGED < <(git diff --name-only --diff-filter=ACMRT "${DIFF_ARGS[@]}" \ - | grep -E '\.(js|cjs|mjs|ts|tsx|json|ya?ml|md)$' || true) - -if [[ ${#CHANGED[@]} -eq 0 ]]; then - echo "No supported files changed; skipping Prettier." - exit 0 -fi - -echo "Running Prettier on ${#CHANGED[@]} file(s):" -printf ' - %s\n' "${CHANGED[@]}" - -yarn run --silent prettier "${CHANGED[@]}" "${PRETTIER_ARGS[@]}" diff --git a/tests/functional/aws-node-sdk/lib/fixtures/project.js b/tests/functional/aws-node-sdk/lib/fixtures/project.js index 3b71a4ff79..1c1a4bbdda 100644 --- a/tests/functional/aws-node-sdk/lib/fixtures/project.js +++ b/tests/functional/aws-node-sdk/lib/fixtures/project.js @@ -10,9 +10,7 @@ const fakeDataSource = { generateManyBucketNames(numberOfBuckets) { const random = Math.round(Math.random() * 100).toString(); - return Array - .from(Array(numberOfBuckets).keys()) - .map(i => `${baseName}-${random}-${i}`); + return Array.from(Array(numberOfBuckets).keys()).map(i => `${baseName}-${random}-${i}`); }, }; diff --git a/tests/functional/aws-node-sdk/lib/json/mem_credentials.json b/tests/functional/aws-node-sdk/lib/json/mem_credentials.json index 66c4599b8e..b55449f72e 100644 --- a/tests/functional/aws-node-sdk/lib/json/mem_credentials.json +++ b/tests/functional/aws-node-sdk/lib/json/mem_credentials.json @@ -1,18 +1,18 @@ { - "default": { - "accessKey": "accessKey1", - "secretKey": "verySecretKey1" - }, - "lisa": { - "accessKey": "accessKey2", - "secretKey": "verySecretKey2" - }, - "replication": { - "accessKey": "replicationKey1", - "secretKey": "replicationSecretKey1" - }, - "vault": { - "accessKey": "TESTAK00000000000000", - "secretKey": "TESTSK0000000000000000000000000000000000" - } + "default": { + "accessKey": "accessKey1", + "secretKey": "verySecretKey1" + }, + "lisa": { + "accessKey": "accessKey2", + "secretKey": "verySecretKey2" + }, + "replication": { + "accessKey": "replicationKey1", + "secretKey": "replicationSecretKey1" + }, + "vault": { + "accessKey": "TESTAK00000000000000", + "secretKey": "TESTSK0000000000000000000000000000000000" + } } diff --git a/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json b/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json index af23d0e8cc..75f09cc09f 100644 --- a/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json +++ b/tests/functional/aws-node-sdk/lib/json/s3c_credentials.json @@ -1,14 +1,14 @@ { - "default": { - "accessKey": "ACC1AK00000000000000", - "secretKey": "ACC1SK0000000000000000000000000000000000" - }, - "lisa": { - "accessKey": "ACC2AK00000000000000", - "secretKey": "ACC2SK0000000000000000000000000000000000" - }, - "replication": { - "accessKey": "ACCREPAK000000000000", - "secretKey": "ACCREPSK00000000000000000000000000000000" - } + "default": { + "accessKey": "ACC1AK00000000000000", + "secretKey": "ACC1SK0000000000000000000000000000000000" + }, + "lisa": { + "accessKey": "ACC2AK00000000000000", + "secretKey": "ACC2SK0000000000000000000000000000000000" + }, + "replication": { + "accessKey": "ACCREPAK000000000000", + "secretKey": "ACCREPSK00000000000000000000000000000000" + } } diff --git a/tests/functional/aws-node-sdk/lib/utility/bucket-util.js b/tests/functional/aws-node-sdk/lib/utility/bucket-util.js index f214540e30..aaa05bf483 100644 --- a/tests/functional/aws-node-sdk/lib/utility/bucket-util.js +++ b/tests/functional/aws-node-sdk/lib/utility/bucket-util.js @@ -21,18 +21,18 @@ class BucketUtility { credentials: { accessKeyId: '', secretAccessKey: '' }, forcePathStyle: true, signer: { sign: async request => request }, - }); - } - else { + }); + } else { this.s3 = new S3Client({ ...s3Config, maxAttempts: 0, - }); - } + }); + } } bucketExists(bucketName) { - return this.s3.send(new HeadBucketCommand({ Bucket: bucketName })) + return this.s3 + .send(new HeadBucketCommand({ Bucket: bucketName })) .then(() => true) .catch(err => { if (err.name === 'NotFound') { @@ -43,7 +43,8 @@ class BucketUtility { } createOne(bucketName) { - return this.s3.send(new CreateBucketCommand({ Bucket: bucketName })) + return this.s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => bucketName) .catch(err => { throw err; @@ -51,16 +52,18 @@ class BucketUtility { } createOneWithLock(bucketName) { - return this.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })).then(() => bucketName); + return this.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => bucketName); } createMany(bucketNames) { - const promises = bucketNames.map(bucketName => - this.createOne(bucketName), - ); + const promises = bucketNames.map(bucketName => this.createOne(bucketName)); return Promise.all(promises); } @@ -69,9 +72,7 @@ class BucketUtility { const bucketName = projectFixture.generateBucketName(); return this.createOne(bucketName); } - const bucketNames = projectFixture - .generateManyBucketNames(nBuckets) - .sort(() => 0.5 - Math.random()); + const bucketNames = projectFixture.generateManyBucketNames(nBuckets).sort(() => 0.5 - Math.random()); return this.createMany(bucketNames); } @@ -80,12 +81,10 @@ class BucketUtility { } deleteMany(bucketNames) { - const promises = bucketNames.map(bucketName => - this.deleteOne(bucketName), - ); + const promises = bucketNames.map(bucketName => this.deleteOne(bucketName)); return Promise.all(promises); } - + /** * Recursively delete all versions of all objects within the bucket * @param bucketName @@ -97,33 +96,36 @@ class BucketUtility { let isTruncated = true; while (isTruncated) { - const response = await this.s3.send(new ListObjectVersionsCommand({ - Bucket: bucketName, - KeyMarker: keyMarker, - VersionIdMarker: versionIdMarker, - })); - - const objects = [ - ...(response.Versions || []), - ...(response.DeleteMarkers || []), - ].map(({ Key, VersionId }) => ({ Key, VersionId })); + const response = await this.s3.send( + new ListObjectVersionsCommand({ + Bucket: bucketName, + KeyMarker: keyMarker, + VersionIdMarker: versionIdMarker, + }), + ); + + const objects = [...(response.Versions || []), ...(response.DeleteMarkers || [])].map( + ({ Key, VersionId }) => ({ Key, VersionId }), + ); if (objects.length > 0) { try { - const result = await this.s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true - }, - ...(BypassGovernanceRetention && { BypassGovernanceRetention }), - })); + const result = await this.s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true, + }, + ...(BypassGovernanceRetention && { BypassGovernanceRetention }), + }), + ); if (result.Errors && result.Errors.length > 0) { for (const e of result.Errors) { // eslint-disable-next-line no-console console.warn( `Warning BucketUtility.empty(): failed to delete s3://${bucketName}/${e.Key}` + - ` (versionId=${e.VersionId}): ${e.Code} - ${e.Message}` + ` (versionId=${e.VersionId}): ${e.Code} - ${e.Message}`, ); } } @@ -133,14 +135,18 @@ class BucketUtility { if (err.name !== 'BadDigest') { throw err; } - await Promise.all(objects.map(({ Key, VersionId }) => - this.s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key, - VersionId, - ...(BypassGovernanceRetention && { BypassGovernanceRetention }), - })) - )); + await Promise.all( + objects.map(({ Key, VersionId }) => + this.s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key, + VersionId, + ...(BypassGovernanceRetention && { BypassGovernanceRetention }), + }), + ), + ), + ); } } @@ -153,12 +159,10 @@ class BucketUtility { } emptyMany(bucketNames) { - const promises = bucketNames.map( - bucketName => this.empty(bucketName) - ); + const promises = bucketNames.map(bucketName => this.empty(bucketName)); return Promise.all(promises); } - + emptyIfExists(bucketName) { return this.bucketExists(bucketName).then(exists => { if (exists) { @@ -169,14 +173,13 @@ class BucketUtility { } emptyManyIfExists(bucketNames) { - const promises = bucketNames.map(bucketName => - this.emptyIfExists(bucketName), - ); + const promises = bucketNames.map(bucketName => this.emptyIfExists(bucketName)); return Promise.all(promises); } getOwner() { - return this.s3.send(new ListBucketsCommand({})) + return this.s3 + .send(new ListBucketsCommand({})) .then(data => data.Owner) .catch(err => { throw err; diff --git a/tests/functional/aws-node-sdk/lib/utility/cors-util.js b/tests/functional/aws-node-sdk/lib/utility/cors-util.js index a1d52bf8cc..9e14d3fe6d 100644 --- a/tests/functional/aws-node-sdk/lib/utility/cors-util.js +++ b/tests/functional/aws-node-sdk/lib/utility/cors-util.js @@ -6,11 +6,9 @@ const conf = require('../../../../../lib/Config').config; const transport = conf.https ? https : http; const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; -const hostname = process.env.AWS_ON_AIR ? 's3.amazonaws.com' : - ipAddress; +const hostname = process.env.AWS_ON_AIR ? 's3.amazonaws.com' : ipAddress; const port = process.env.AWS_ON_AIR ? 80 : 8000; - const statusCode = { 200: 200, 301: 301, // website redirect @@ -25,11 +23,10 @@ const statusCode = { }; function methodRequest(params, callback) { - const { method, bucket, objectKey, query, headers, code, - headersResponse, headersOmitted, isWebsite } = params; - const websiteHostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; + const { method, bucket, objectKey, query, headers, code, headersResponse, headersOmitted, isWebsite } = params; + const websiteHostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; const options = { port, @@ -59,32 +56,29 @@ function methodRequest(params, callback) { res.on('end', () => { const total = body.join(''); if (code) { - const message = Number.isNaN(parseInt(code, 10)) ? - `${code}` : ''; + const message = Number.isNaN(parseInt(code, 10)) ? `${code}` : ''; assert(total.indexOf(message) > -1, `Expected ${message}`); - assert.deepEqual(res.statusCode, statusCode[code], - `status code expected: ${statusCode[code]}`); + assert.deepEqual(res.statusCode, statusCode[code], `status code expected: ${statusCode[code]}`); } if (headersResponse) { Object.keys(headersResponse).forEach(key => { - assert.deepEqual(res.headers[key], headersResponse[key], - `error header: ${key}`); + assert.deepEqual(res.headers[key], headersResponse[key], `error header: ${key}`); }); } else { - // if no headersResponse provided, should not have these headers - // in the request - ['access-control-allow-origin', + // if no headersResponse provided, should not have these headers + // in the request + [ + 'access-control-allow-origin', 'access-control-allow-methods', 'access-control-allow-credentials', - 'vary'].forEach(key => { - assert.strictEqual(res.headers[key], undefined, - `Error: ${key} should not have value`); - }); + 'vary', + ].forEach(key => { + assert.strictEqual(res.headers[key], undefined, `Error: ${key} should not have value`); + }); } if (headersOmitted) { headersOmitted.forEach(key => { - assert.strictEqual(res.headers[key], undefined, - `Error: ${key} should not have value`); + assert.strictEqual(res.headers[key], undefined, `Error: ${key} should not have value`); }); } return callback(); diff --git a/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js b/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js index efb678f9c8..0b798d9140 100644 --- a/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js +++ b/tests/functional/aws-node-sdk/lib/utility/createEncryptedBucket.js @@ -13,27 +13,29 @@ function safeJSONParse(s) { } function createEncryptedBucket(bucketParams, cb) { - process.stdout.write('Creating encrypted bucket' + - `${bucketParams.Bucket}`); + process.stdout.write('Creating encrypted bucket' + `${bucketParams.Bucket}`); const config = getConfig(); const endpointWithoutHttp = config.endpoint.split('//')[1]; const host = endpointWithoutHttp.split(':')[0]; const port = endpointWithoutHttp.split(':')[1]; let locationConstraint; - if (bucketParams.CreateBucketConfiguration && - bucketParams.CreateBucketConfiguration.LocationConstraint) { - locationConstraint = bucketParams.CreateBucketConfiguration - .LocationConstraint; + if (bucketParams.CreateBucketConfiguration && bucketParams.CreateBucketConfiguration.LocationConstraint) { + locationConstraint = bucketParams.CreateBucketConfiguration.LocationConstraint; } const prog = `${__dirname}/../../../../../bin/create_encrypted_bucket.js`; let args = [ prog, - '-a', config.credentials.accessKeyId, - '-k', config.credentials.secretAccessKey, - '-b', bucketParams.Bucket, - '-h', host, - '-p', port, + '-a', + config.credentials.accessKeyId, + '-k', + config.credentials.secretAccessKey, + '-b', + bucketParams.Bucket, + '-h', + host, + '-p', + port, '-v', ]; if (locationConstraint) { @@ -43,24 +45,27 @@ function createEncryptedBucket(bucketParams, cb) { args = args.concat('-s'); } const body = []; - const child = childProcess.spawn(args[0], args) - .on('exit', () => { - const hasSucceed = body.join('').split('\n').find(item => { - const json = safeJSONParse(item); - const test = !(json instanceof Error) && json.name === 'S3' && - json.statusCode === 200; - if (test) { - return true; + const child = childProcess + .spawn(args[0], args) + .on('exit', () => { + const hasSucceed = body + .join('') + .split('\n') + .find(item => { + const json = safeJSONParse(item); + const test = !(json instanceof Error) && json.name === 'S3' && json.statusCode === 200; + if (test) { + return true; + } + return false; + }); + if (!hasSucceed) { + process.stderr.write(`${body.join('')}\n`); + return cb(new Error('Cannot create encrypted bucket')); } - return false; - }); - if (!hasSucceed) { - process.stderr.write(`${body.join('')}\n`); - return cb(new Error('Cannot create encrypted bucket')); - } - return cb(); - }) - .on('error', cb); + return cb(); + }) + .on('error', cb); child.stdout.on('data', chunk => body.push(chunk.toString())); } diff --git a/tests/functional/aws-node-sdk/lib/utility/customS3Request.js b/tests/functional/aws-node-sdk/lib/utility/customS3Request.js index d9c748cdd6..950106d303 100644 --- a/tests/functional/aws-node-sdk/lib/utility/customS3Request.js +++ b/tests/functional/aws-node-sdk/lib/utility/customS3Request.js @@ -6,7 +6,6 @@ const getConfig = require('../../test/support/config'); const config = getConfig('default'); const customRequestMiddleware = buildParams => next => async args => { - const { headers, query } = buildParams; const prevReq = args.request; @@ -35,10 +34,11 @@ const customRequestMiddleware = buildParams => next => async args => { async function customS3Request(CommandClass, params, buildParams) { const customS3 = new S3Client({ ...config }); - customS3.middlewareStack.add( - customRequestMiddleware(buildParams), - { step: 'build', name: 'customRequestMiddleware', tags: ['CUSTOM'] } - ); + customS3.middlewareStack.add(customRequestMiddleware(buildParams), { + step: 'build', + name: 'customRequestMiddleware', + tags: ['CUSTOM'], + }); const command = new CommandClass(params); const response = await customS3.send(command); @@ -50,7 +50,6 @@ async function customS3Request(CommandClass, params, buildParams) { }; return resData; - } module.exports = customS3Request; diff --git a/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js b/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js index d2ebeff178..65e402ac47 100644 --- a/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js +++ b/tests/functional/aws-node-sdk/lib/utility/genMaxSizeMetaHeaders.js @@ -3,12 +3,10 @@ const constants = require('../../../../../constants'); function genMaxSizeMetaHeaders() { const metaHeaders = {}; const counter = 8; - const bytesPerHeader = - (constants.maximumMetaHeadersSize / counter); + const bytesPerHeader = constants.maximumMetaHeadersSize / counter; for (let i = 0; i < counter; i++) { const key = `header${i}`; - const valueLength = bytesPerHeader - - ('x-amz-meta-'.length + key.length); + const valueLength = bytesPerHeader - ('x-amz-meta-'.length + key.length); metaHeaders[key] = '0'.repeat(valueLength); } return metaHeaders; diff --git a/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js b/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js index 9e508f0c44..be9196832f 100644 --- a/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js +++ b/tests/functional/aws-node-sdk/lib/utility/provideRawOutput.js @@ -27,15 +27,13 @@ function provideRawOutput(args, cb) { httpCode = lines.find(line => { const trimmed = line.trim().toUpperCase(); // ignore 100 Continue HTTP code - if (trimmed.startsWith('HTTP/1.1 ') && - !trimmed.includes('100 CONTINUE')) { + if (trimmed.startsWith('HTTP/1.1 ') && !trimmed.includes('100 CONTINUE')) { return true; } return false; }); if (httpCode) { - httpCode = httpCode.trim().replace('HTTP/1.1 ', '') - .toUpperCase(); + httpCode = httpCode.trim().replace('HTTP/1.1 ', '').toUpperCase(); } } return cb(httpCode, procData); @@ -46,8 +44,6 @@ function provideRawOutput(args, cb) { } provideRawOutput[util.promisify.custom] = args => - new Promise(resolve => - provideRawOutput(args, (httpCode, rawOutput) => resolve({ httpCode, rawOutput })) - ); + new Promise(resolve => provideRawOutput(args, (httpCode, rawOutput) => resolve({ httpCode, rawOutput }))); module.exports = provideRawOutput; diff --git a/tests/functional/aws-node-sdk/lib/utility/replication.js b/tests/functional/aws-node-sdk/lib/utility/replication.js index 7f1f466af9..f50cefbe0c 100644 --- a/tests/functional/aws-node-sdk/lib/utility/replication.js +++ b/tests/functional/aws-node-sdk/lib/utility/replication.js @@ -1,16 +1,6 @@ const replicationUtils = { - requiredConfigProperties: [ - 'Role', - 'Rules', - 'Status', - 'Destination', - 'Bucket', - ], - optionalConfigProperties: [ - 'ID', - 'StorageClass', - 'Prefix', - ], + requiredConfigProperties: ['Role', 'Rules', 'Status', 'Destination', 'Bucket'], + optionalConfigProperties: ['ID', 'StorageClass', 'Prefix'], invalidRoleARNs: [ '', '*:aws:iam::account-id:role/resource', @@ -33,7 +23,6 @@ const replicationUtils = { 'arn:aws:iam::ac:role', 'arn:aws:iam::a c:role', 'arn:aws:iam::*:role', - ], invalidBucketARNs: [ '', @@ -46,18 +35,9 @@ const replicationUtils = { 'arn:aws:s3:::*', 'arn:aws:s3:::invalidBucketName', ], - validStatuses: [ - 'Enabled', - 'Disabled', - ], - validStorageClasses: [ - 'STANDARD', - 'STANDARD_IA', - 'REDUCED_REDUNDANCY', - ], - validMultipleStorageClasses: [ - 'zenko,us-east-2', - ], + validStatuses: ['Enabled', 'Disabled'], + validStorageClasses: ['STANDARD', 'STANDARD_IA', 'REDUCED_REDUNDANCY'], + validMultipleStorageClasses: ['zenko,us-east-2'], }; module.exports = replicationUtils; diff --git a/tests/functional/aws-node-sdk/lib/utility/tagging.js b/tests/functional/aws-node-sdk/lib/utility/tagging.js index 9039a9b5eb..f215fcb9fc 100644 --- a/tests/functional/aws-node-sdk/lib/utility/tagging.js +++ b/tests/functional/aws-node-sdk/lib/utility/tagging.js @@ -1,13 +1,14 @@ const taggingTests = [ - { tag: { key: '+- =._:/', value: '+- =._:/' }, - it: 'should return tags if tags are valid' }, - { tag: { key: 'key1', value: '' }, - it: 'should return tags if value is an empty string' }, - { tag: { key: 'w'.repeat(129), value: 'foo' }, + { tag: { key: '+- =._:/', value: '+- =._:/' }, it: 'should return tags if tags are valid' }, + { tag: { key: 'key1', value: '' }, it: 'should return tags if value is an empty string' }, + { + tag: { key: 'w'.repeat(129), value: 'foo' }, error: 'InvalidTag', code: 400, - it: 'should return InvalidTag if key length is greater than 128' }, - { tag: { key: 'bar', value: 'f'.repeat(257) }, + it: 'should return InvalidTag if key length is greater than 128', + }, + { + tag: { key: 'bar', value: 'f'.repeat(257) }, error: 'InvalidTag', code: 400, it: 'should return InvalidTag if key length is greater than 256', diff --git a/tests/functional/aws-node-sdk/lib/utility/test-utils.js b/tests/functional/aws-node-sdk/lib/utility/test-utils.js index 8e90ab8e95..f4e8c7cdc4 100644 --- a/tests/functional/aws-node-sdk/lib/utility/test-utils.js +++ b/tests/functional/aws-node-sdk/lib/utility/test-utils.js @@ -15,7 +15,7 @@ function hasLocation(lc) { * In our implementation, it's the only way to move objects to cold storage (can't write directly to cold) * Even if Transition could be used without cold storage location (hot transition), but both features are * enabled together at the moment. -*/ + */ const hasColdStorage = config.supportedLifecycleRules.some(rule => rule.endsWith('Transition')); module.exports = { diff --git a/tests/functional/aws-node-sdk/lib/utility/website-util.js b/tests/functional/aws-node-sdk/lib/utility/website-util.js index 463403b525..605112823d 100644 --- a/tests/functional/aws-node-sdk/lib/utility/website-util.js +++ b/tests/functional/aws-node-sdk/lib/utility/website-util.js @@ -3,16 +3,17 @@ const async = require('async'); const fs = require('fs'); const path = require('path'); const url = require('url'); -const { CreateBucketCommand, - DeleteBucketCommand, +const { + CreateBucketCommand, + DeleteBucketCommand, PutBucketWebsiteCommand, - DeleteObjectCommand, - PutObjectCommand } = require('@aws-sdk/client-s3'); + DeleteObjectCommand, + PutObjectCommand, +} = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../../raw-node/utils/makeRequest'); -const bucketName = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; +const bucketName = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; let awsCredentials; function _parseConfigValue(string, fileSlice) { @@ -36,10 +37,8 @@ function _retrieveAWSCredentials(profile) { const fileContents = file.split('\n'); const profileIndex = file.indexOf(`[${profile}]`); if (profileIndex > -1) { - const accessKey = _parseConfigValue('aws_access_key_id', - fileContents.slice(profileIndex)); - const secretKey = _parseConfigValue('aws_secret_access_key', - fileContents.slice(profileIndex)); + const accessKey = _parseConfigValue('aws_access_key_id', fileContents.slice(profileIndex)); + const secretKey = _parseConfigValue('aws_secret_access_key', fileContents.slice(profileIndex)); return { accessKey, secretKey }; } const msg = `Profile ${profile} does not exist in AWS credential file`; @@ -75,46 +74,41 @@ function _assertResponseHtml(response, elemtag, content) { const startIndex = response.indexOf(startingTag); const endIndex = response.indexOf(''); assert(startIndex > -1 && endIndex > -1, 'Did not find ul element'); - const ulElem = response.slice(startIndex + startingTag.length, - endIndex); + const ulElem = response.slice(startIndex + startingTag.length, endIndex); content.forEach(item => { _assertResponseHtml(ulElem, 'li', item); }); } else { const elem = `<${elemtag}>${content}`; - assert(response.includes(elem), - `Expected but did not find '${elem}' in html`); + assert(response.includes(elem), `Expected but did not find '${elem}' in html`); } } function _assertContainsHtml(responseBody) { - assert(responseBody.startsWith('') && - responseBody.includes(''), 'Did not find html tags'); + assert(responseBody.startsWith('') && responseBody.includes(''), 'Did not find html tags'); } function _assertResponseHtml404(method, response, type) { assert.strictEqual(response.statusCode, 404); if (method === 'HEAD') { if (type === '404-no-such-bucket') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'NoSuchBucket'); + assert.strictEqual(response.headers['x-amz-error-code'], 'NoSuchBucket'); // Need arsenal fixed to remove period at the end // so compatible with aws - assert.strictEqual(response.headers['x-amz-error-message'], - 'The specified bucket does not exist.'); + assert.strictEqual(response.headers['x-amz-error-message'], 'The specified bucket does not exist.'); } else if (type === '404-no-such-website-configuration') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'NoSuchWebsiteConfiguration'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'The specified bucket does not have a website configuration'); + assert.strictEqual(response.headers['x-amz-error-code'], 'NoSuchWebsiteConfiguration'); + assert.strictEqual( + response.headers['x-amz-error-message'], + 'The specified bucket does not have a website configuration', + ); } else if (type === '404-not-found') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'NoSuchKey'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'The specified key does not exist.'); + assert.strictEqual(response.headers['x-amz-error-code'], 'NoSuchKey'); + assert.strictEqual(response.headers['x-amz-error-message'], 'The specified key does not exist.'); } else { - throw new Error(`'${type}' is not a recognized 404 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 404 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } // don't need to check HTML for head requests return; @@ -131,18 +125,15 @@ function _assertResponseHtml404(method, response, type) { } else if (type === '404-no-such-website-configuration') { _assertResponseHtml(response.body, 'ul', [ 'Code: NoSuchWebsiteConfiguration', - 'Message: The specified bucket does not have a ' + - 'website configuration', + 'Message: The specified bucket does not have a ' + 'website configuration', `BucketName: ${bucketName}`, ]); } else if (type === '404-not-found') { - _assertResponseHtml(response.body, 'ul', [ - 'Code: NoSuchKey', - 'Message: The specified key does not exist.', - ]); + _assertResponseHtml(response.body, 'ul', ['Code: NoSuchKey', 'Message: The specified key does not exist.']); } else { - throw new Error(`'${type}' is not a recognized 404 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 404 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } } @@ -150,38 +141,35 @@ function _assertResponseHtml403(method, response, type) { assert.strictEqual(response.statusCode, 403); if (method === 'HEAD') { if (type === '403-access-denied') { - assert.strictEqual(response.headers['x-amz-error-code'], - 'AccessDenied'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'Access Denied'); + assert.strictEqual(response.headers['x-amz-error-code'], 'AccessDenied'); + assert.strictEqual(response.headers['x-amz-error-message'], 'Access Denied'); } else if (type !== '403-retrieve-error-document') { - throw new Error(`'${type}' is not a recognized 403 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 403 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } } else { _assertContainsHtml(response.body); _assertResponseHtml(response.body, 'title', '403 Forbidden'); _assertResponseHtml(response.body, 'h1', '403 Forbidden'); - _assertResponseHtml(response.body, 'ul', [ - 'Code: AccessDenied', - 'Message: Access Denied', - ]); + _assertResponseHtml(response.body, 'ul', ['Code: AccessDenied', 'Message: Access Denied']); if (type === '403-retrieve-error-document') { - _assertResponseHtml(response.body, 'h3', - 'An Error Occurred While Attempting to ' + - 'Retrieve a Custom Error Document'); + _assertResponseHtml( + response.body, + 'h3', + 'An Error Occurred While Attempting to ' + 'Retrieve a Custom Error Document', + ); // start searching for second `ul` element after `h3` element const startingTag = ''; - const startIndex = response.body.indexOf(startingTag) - + startingTag.length; - _assertResponseHtml(response.body.slice(startIndex), - 'ul', [ + const startIndex = response.body.indexOf(startingTag) + startingTag.length; + _assertResponseHtml(response.body.slice(startIndex), 'ul', [ 'Code: AccessDenied', 'Message: Access Denied', ]); } else if (type !== '403-access-denied') { - throw new Error(`'${type}' is not a recognized 403 ` + - 'error checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized 403 ` + 'error checked in the WebsiteConfigTester.checkHTML function', + ); } } } @@ -192,22 +180,17 @@ function _assertResponseHtmlErrorUser(response, type) { } else if (type === 'error-user-404') { assert.strictEqual(response.statusCode, 404); } - _assertResponseHtml(response.body, 'title', - 'Error!!'); - _assertResponseHtml(response.body, 'h1', - 'It appears you messed up'); + _assertResponseHtml(response.body, 'title', 'Error!!'); + _assertResponseHtml(response.body, 'h1', 'It appears you messed up'); } function _assertResponseHtmlIndexUser(response) { assert.strictEqual(response.statusCode, 200); - _assertResponseHtml(response.body, 'title', - 'Best testing website ever'); - _assertResponseHtml(response.body, 'h1', 'Welcome to my ' + - 'extraordinary bucket website testing page'); + _assertResponseHtml(response.body, 'title', 'Best testing website ever'); + _assertResponseHtml(response.body, 'h1', 'Welcome to my ' + 'extraordinary bucket website testing page'); } -function _assertResponseHtmlRedirect(response, type, redirectUrl, method, - expectedHeaders) { +function _assertResponseHtmlRedirect(response, type, redirectUrl, method, expectedHeaders) { if (type === 'redirect' || type === 'redirect-user') { assert.strictEqual(response.statusCode, 301); assert.strictEqual(response.body, ''); @@ -218,13 +201,10 @@ function _assertResponseHtmlRedirect(response, type, redirectUrl, method, return; // no need to check HTML } - _assertResponseHtml(response.body, 'title', - 'Best redirect link ever'); - _assertResponseHtml(response.body, 'h1', - 'Welcome to your redirection file'); + _assertResponseHtml(response.body, 'title', 'Best redirect link ever'); + _assertResponseHtml(response.body, 'h1', 'Welcome to your redirection file'); } else if (type.startsWith('redirect-error')) { - assert.strictEqual(response.statusCode, - type === 'redirect-error-found' ? 302 : 301); + assert.strictEqual(response.statusCode, type === 'redirect-error-found' ? 302 : 301); assert.strictEqual(response.headers.location, redirectUrl); Object.entries(expectedHeaders || {}).forEach(([key, val]) => { assert.strictEqual(response.headers[key], val); @@ -232,21 +212,18 @@ function _assertResponseHtmlRedirect(response, type, redirectUrl, method, if (type === 'redirect-error-found') { assert.strictEqual(response.headers['x-amz-error-code'], 'Found'); - assert.strictEqual(response.headers['x-amz-error-message'], - 'Resource Found'); + assert.strictEqual(response.headers['x-amz-error-message'], 'Resource Found'); _assertContainsHtml(response.body); _assertResponseHtml(response.body, 'title', '302 Found'); _assertResponseHtml(response.body, 'h1', '302 Found'); - _assertResponseHtml(response.body, 'ul', [ - 'Code: Found', - 'Message: Resource Found', - ]); + _assertResponseHtml(response.body, 'ul', ['Code: Found', 'Message: Resource Found']); } else { _assertResponseHtmlErrorUser(response, type); } } else { - throw new Error(`'${type}' is not a recognized redirect type ` + - 'checked in the WebsiteConfigTester.checkHTML function'); + throw new Error( + `'${type}' is not a recognized redirect type ` + 'checked in the WebsiteConfigTester.checkHTML function', + ); } } @@ -285,21 +262,20 @@ class WebsiteConfigTester { } /** checkHTML - check response for website head or get - * @param {object} params - function params - * @param {string} params.method - type of website request, 'HEAD' or 'GET' - * @param {string} params.responseType - type of response expected - * @param {string} [params.auth] - whether to use valid or invalid auth - * crendentials: 'valid credentials' or 'invalid credentials' - * @param {string} [params.url] - request url - * @param {string} [params.redirectUrl] - redirect - * @param {object} [params.expectedHeaders] - expected headers in response - * with expected values (e.g., {x-amz-error-code: AccessDenied}) - * @param {function} callback - callback - * @return {undefined} - */ + * @param {object} params - function params + * @param {string} params.method - type of website request, 'HEAD' or 'GET' + * @param {string} params.responseType - type of response expected + * @param {string} [params.auth] - whether to use valid or invalid auth + * crendentials: 'valid credentials' or 'invalid credentials' + * @param {string} [params.url] - request url + * @param {string} [params.redirectUrl] - redirect + * @param {object} [params.expectedHeaders] - expected headers in response + * with expected values (e.g., {x-amz-error-code: AccessDenied}) + * @param {function} callback - callback + * @return {undefined} + */ static checkHTML(params, callback) { - const { method, responseType, auth, url, redirectUrl, expectedHeaders } - = params; + const { method, responseType, auth, url, redirectUrl, expectedHeaders } = params; _makeWebsiteRequest(auth, method, url, (err, res) => { assert.strictEqual(err, null, `Unexpected request err ${err}`); if (responseType) { @@ -310,20 +286,20 @@ class WebsiteConfigTester { } else if (responseType.startsWith('error-user')) { _assertResponseHtmlErrorUser(res, responseType); } else if (responseType.startsWith('redirect')) { - _assertResponseHtmlRedirect(res, responseType, redirectUrl, - method, expectedHeaders); + _assertResponseHtmlRedirect(res, responseType, redirectUrl, method, expectedHeaders); if (responseType === 'redirect-user') { process.stdout.write('Following redirect location\n'); - return this.checkHTML({ method, - url: res.headers.location, - responseType: 'redirected-user' }, - callback); + return this.checkHTML( + { method, url: res.headers.location, responseType: 'redirected-user' }, + callback, + ); } } else if (responseType === 'index-user') { _assertResponseHtmlIndexUser(res); } else { - throw new Error(`'${responseType}' is not a response ` + - 'type recognized by WebsiteConfigTester.checkHTML'); + throw new Error( + `'${responseType}' is not a response ` + 'type recognized by WebsiteConfigTester.checkHTML', + ); } } return callback(); @@ -341,48 +317,67 @@ class WebsiteConfigTester { * @param {function} cb - callback to end test * @return {undefined} */ - static makeHeadRequest(auth, url, expectedStatusCode, expectedHeaders, - cb) { + static makeHeadRequest(auth, url, expectedStatusCode, expectedHeaders, cb) { _makeWebsiteRequest(auth, 'HEAD', url, (err, res) => { // body should be empty assert.deepStrictEqual(res.body, ''); assert.strictEqual(res.statusCode, expectedStatusCode); const headers = Object.keys(expectedHeaders); headers.forEach(header => { - assert.strictEqual(res.headers[header], - expectedHeaders[header]); + assert.strictEqual(res.headers[header], expectedHeaders[header]); }); return cb(); }); } static createPutBucketWebsite(s3, bucket, bucketACL, objects, done) { - s3.send(new CreateBucketCommand({ Bucket: bucket, ACL: bucketACL })).then(() => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - return s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).then(() => async.forEachOf(objects, - (acl, object, next) => { - s3.send(new PutObjectCommand({ Bucket: bucket, - Key: `${object}.html`, - ACL: acl, - Body: fs.readFileSync(path.join(__dirname, - `/../../test/object/websiteFiles/${object}.html`)), - })).then(() => next()).catch(next); - }, done)); - }).catch(err => done(err)); + s3.send(new CreateBucketCommand({ Bucket: bucket, ACL: bucketACL })) + .then(() => { + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + return s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .then(() => + async.forEachOf( + objects, + (acl, object, next) => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: `${object}.html`, + ACL: acl, + Body: fs.readFileSync( + path.join(__dirname, `/../../test/object/websiteFiles/${object}.html`), + ), + }), + ) + .then(() => next()) + .catch(next); + }, + done, + ), + ); + }) + .catch(err => done(err)); } static deleteObjectsThenBucket(s3, bucket, objects, done) { - async.forEachOf(objects, (acl, object, next) => { - s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: `${object}.html` })).then(() => next()).catch(next); - }, err => { - if (err) { - return done(err); - } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })).then(() => done()).catch(done); - }); + async.forEachOf( + objects, + (acl, object, next) => { + s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: `${object}.html` })) + .then(() => next()) + .catch(next); + }, + err => { + if (err) { + return done(err); + } + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); + }, + ); } } diff --git a/tests/functional/aws-node-sdk/schema/bucket.json b/tests/functional/aws-node-sdk/schema/bucket.json index ee69950078..1d6b3933cd 100644 --- a/tests/functional/aws-node-sdk/schema/bucket.json +++ b/tests/functional/aws-node-sdk/schema/bucket.json @@ -1,94 +1,79 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://jsonschema.net", - "type": "object", - "properties": { - "Contents": { - "id": "http://jsonschema.net/Contents", - "type": "array", - "minItems": 0, - "items": { - "id": "http://jsonschema.net/Buckets/0", - "type": "object", - "properties": { - "Key": { - "id": "http://jsonschema.net/Contents/0/Key", + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "http://jsonschema.net", + "type": "object", + "properties": { + "Contents": { + "id": "http://jsonschema.net/Contents", + "type": "array", + "minItems": 0, + "items": { + "id": "http://jsonschema.net/Buckets/0", + "type": "object", + "properties": { + "Key": { + "id": "http://jsonschema.net/Contents/0/Key", + "type": "string" + }, + "LastModified": { + "id": "http://jsonschema.net/Contents/0/LastModified", + "type": "object" + }, + "ETag": { + "id": "http://jsonschema.net/Contents/0/ETag", + "type": "string" + }, + "Size": { + "id": "http://jsonschema.net/Contents/0/Size", + "type": "integer" + }, + "StorageClass": { + "id": "http://jsonschema.net/Contents/0/StorageClass", + "enum": ["STANDARD", "REDUCED_REDUNDANCY", "GLACIER"] + }, + "Owner": { + "id": "http://jsonschema.net/Contents/0/Owner", + "type": "object", + "properties": { + "DisplayName": { + "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", + "type": "string" + }, + "ID": { + "id": "http://jsonschema.net/Contents/0/Owner/ID", + "type": "string" + } + }, + "required": ["DisplayName", "ID"] + } + }, + "required": ["Key", "LastModified", "ETag", "Size", "StorageClass", "Owner"] + } + }, + "Marker": { + "id": "http://jsonschema.net/Marker", + "type": "string" + }, + "Name": { + "id": "http://jsonschema.net/Name", "type": "string" - }, - "LastModified": { - "id": "http://jsonschema.net/Contents/0/LastModified", - "type": "object" - }, - "ETag": { - "id": "http://jsonschema.net/Contents/0/ETag", + }, + "Prefix": { + "id": "http://jsonschema.net/Prefix", "type": "string" - }, - "Size": { - "id": "http://jsonschema.net/Contents/0/Size", + }, + "MaxKeys": { + "id": "http://jsonschema.net/MaxKeys", "type": "integer" - }, - "StorageClass": { - "id": "http://jsonschema.net/Contents/0/StorageClass", - "enum": [ "STANDARD", "REDUCED_REDUNDANCY", "GLACIER" ] - }, - "Owner": { - "id": "http://jsonschema.net/Contents/0/Owner", - "type": "object", - "properties": { - "DisplayName": { - "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", - "type": "string" - }, - "ID": { - "id": "http://jsonschema.net/Contents/0/Owner/ID", - "type": "string" - } - }, - "required": [ "DisplayName", "ID" ] - } }, - "required": [ - "Key", - "LastModified", - "ETag", - "Size", - "StorageClass", - "Owner" - ] - } - }, - "Marker": { - "id": "http://jsonschema.net/Marker", - "type": "string" - }, - "Name": { - "id": "http://jsonschema.net/Name", - "type": "string" - }, - "Prefix": { - "id": "http://jsonschema.net/Prefix", - "type": "string" - }, - "MaxKeys": { - "id": "http://jsonschema.net/MaxKeys", - "type": "integer" - }, - "CommonPrefixes": { - "id": "http://jsonschema.net/CommonPrefixes", - "type": "array" + "CommonPrefixes": { + "id": "http://jsonschema.net/CommonPrefixes", + "type": "array" + }, + "IsTruncated": { + "id": "http://jsonschema.net/IsTruncated", + "type": "boolean" + } }, - "IsTruncated": { - "id": "http://jsonschema.net/IsTruncated", - "type": "boolean" - } - }, - "required": [ - "IsTruncated", - "Marker", - "Contents", - "Name", - "Prefix", - "MaxKeys", - "CommonPrefixes" - ] + "required": ["IsTruncated", "Marker", "Contents", "Name", "Prefix", "MaxKeys", "CommonPrefixes"] } diff --git a/tests/functional/aws-node-sdk/schema/bucketV2.json b/tests/functional/aws-node-sdk/schema/bucketV2.json index c868be3318..0594061d1c 100644 --- a/tests/functional/aws-node-sdk/schema/bucketV2.json +++ b/tests/functional/aws-node-sdk/schema/bucketV2.json @@ -1,104 +1,91 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://jsonschema.net", - "type": "object", - "properties": { - "Contents": { - "id": "http://jsonschema.net/Contents", - "type": "array", - "minItems": 0, - "items": { - "id": "http://jsonschema.net/Buckets/0", - "type": "object", - "properties": { - "Key": { - "id": "http://jsonschema.net/Contents/0/Key", + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "http://jsonschema.net", + "type": "object", + "properties": { + "Contents": { + "id": "http://jsonschema.net/Contents", + "type": "array", + "minItems": 0, + "items": { + "id": "http://jsonschema.net/Buckets/0", + "type": "object", + "properties": { + "Key": { + "id": "http://jsonschema.net/Contents/0/Key", + "type": "string" + }, + "LastModified": { + "id": "http://jsonschema.net/Contents/0/LastModified", + "type": "object" + }, + "ETag": { + "id": "http://jsonschema.net/Contents/0/ETag", + "type": "string" + }, + "Size": { + "id": "http://jsonschema.net/Contents/0/Size", + "type": "integer" + }, + "StorageClass": { + "id": "http://jsonschema.net/Contents/0/StorageClass", + "enum": ["STANDARD", "REDUCED_REDUNDANCY", "GLACIER"] + }, + "Owner": { + "id": "http://jsonschema.net/Contents/0/Owner", + "type": "object", + "properties": { + "DisplayName": { + "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", + "type": "string" + }, + "ID": { + "id": "http://jsonschema.net/Contents/0/Owner/ID", + "type": "string" + } + }, + "required": ["DisplayName", "ID"] + } + }, + "required": ["Key", "LastModified", "ETag", "Size", "StorageClass"] + } + }, + "StartAfter": { + "id": "http://jsonschema.net/StartAfter", + "type": "string" + }, + "ContinuationToken": { + "id": "http://jsonschema.net/ContinuationToken", + "type": "string" + }, + "NextContinuationToken": { + "id": "http://jsonschema.net/NextContinuationToken", "type": "string" - }, - "LastModified": { - "id": "http://jsonschema.net/Contents/0/LastModified", - "type": "object" - }, - "ETag": { - "id": "http://jsonschema.net/Contents/0/ETag", + }, + "Name": { + "id": "http://jsonschema.net/Name", + "type": "string" + }, + "Prefix": { + "id": "http://jsonschema.net/Prefix", "type": "string" - }, - "Size": { - "id": "http://jsonschema.net/Contents/0/Size", + }, + "KeyCount": { + "id": "http://jsonschema.net/KeyCount", "type": "integer" - }, - "StorageClass": { - "id": "http://jsonschema.net/Contents/0/StorageClass", - "enum": [ "STANDARD", "REDUCED_REDUNDANCY", "GLACIER" ] - }, - "Owner": { - "id": "http://jsonschema.net/Contents/0/Owner", - "type": "object", - "properties": { - "DisplayName": { - "id": "http://jsonschema.net/Contents/0/Owner/DisplayName", - "type": "string" - }, - "ID": { - "id": "http://jsonschema.net/Contents/0/Owner/ID", - "type": "string" - } - }, - "required": [ "DisplayName", "ID" ] - } }, - "required": [ - "Key", - "LastModified", - "ETag", - "Size", - "StorageClass" - ] - } - }, - "StartAfter": { - "id": "http://jsonschema.net/StartAfter", - "type": "string" - }, - "ContinuationToken": { - "id": "http://jsonschema.net/ContinuationToken", - "type": "string" - }, - "NextContinuationToken": { - "id": "http://jsonschema.net/NextContinuationToken", - "type": "string" - }, - "Name": { - "id": "http://jsonschema.net/Name", - "type": "string" - }, - "Prefix": { - "id": "http://jsonschema.net/Prefix", - "type": "string" - }, - "KeyCount": { - "id": "http://jsonschema.net/KeyCount", - "type": "integer" - }, - "MaxKeys": { - "id": "http://jsonschema.net/MaxKeys", - "type": "integer" - }, - "CommonPrefixes": { - "id": "http://jsonschema.net/CommonPrefixes", - "type": "array" + "MaxKeys": { + "id": "http://jsonschema.net/MaxKeys", + "type": "integer" + }, + "CommonPrefixes": { + "id": "http://jsonschema.net/CommonPrefixes", + "type": "array" + }, + "IsTruncated": { + "id": "http://jsonschema.net/IsTruncated", + "type": "boolean" + } }, - "IsTruncated": { - "id": "http://jsonschema.net/IsTruncated", - "type": "boolean" - } - }, - "required": [ - "IsTruncated", - "Contents", - "Name", - "Prefix", - "MaxKeys", - "CommonPrefixes" - ] + "required": ["IsTruncated", "Contents", "Name", "Prefix", "MaxKeys", "CommonPrefixes"] } diff --git a/tests/functional/aws-node-sdk/schema/service.json b/tests/functional/aws-node-sdk/schema/service.json index 03b3c1daa6..8cb0a54b1c 100644 --- a/tests/functional/aws-node-sdk/schema/service.json +++ b/tests/functional/aws-node-sdk/schema/service.json @@ -1,48 +1,42 @@ { - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "http://jsonschema.net", - "type": "object", - "properties": { - "Buckets": { - "id": "http://jsonschema.net/Buckets", - "type": "array", - "minItems": 0, - "items": { - "id": "http://jsonschema.net/Buckets/0", - "type": "object", - "properties": { - "Name": { - "id": "http://jsonschema.net/Buckets/0/Name", - "type": "string" - }, - "CreationDate": { - "id": "http://jsonschema.net/Buckets/0/CreationDate", - "type": "object" - } + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "http://jsonschema.net", + "type": "object", + "properties": { + "Buckets": { + "id": "http://jsonschema.net/Buckets", + "type": "array", + "minItems": 0, + "items": { + "id": "http://jsonschema.net/Buckets/0", + "type": "object", + "properties": { + "Name": { + "id": "http://jsonschema.net/Buckets/0/Name", + "type": "string" + }, + "CreationDate": { + "id": "http://jsonschema.net/Buckets/0/CreationDate", + "type": "object" + } + }, + "required": ["Name", "CreationDate"] + } }, - "required": [ - "Name", - "CreationDate" - ] - } - }, - "Owner": { - "id": "http://jsonschema.net/Owner", - "type": "object", - "properties": { - "DisplayName": { - "id": "http://jsonschema.net/Owner/DisplayName", - "type": "string" - }, - "ID": { - "id": "http://jsonschema.net/Owner/ID", - "type": "string" + "Owner": { + "id": "http://jsonschema.net/Owner", + "type": "object", + "properties": { + "DisplayName": { + "id": "http://jsonschema.net/Owner/DisplayName", + "type": "string" + }, + "ID": { + "id": "http://jsonschema.net/Owner/ID", + "type": "string" + } + } } - } - } - }, - "required": [ - "Buckets", - "Owner" - ] + }, + "required": ["Buckets", "Owner"] } diff --git a/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js b/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js index 88d0f72027..42b445bc09 100644 --- a/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js +++ b/tests/functional/aws-node-sdk/test/bucket/aclUsingPredefinedGroups.js @@ -31,7 +31,7 @@ withV4(sigCfg => { return otherAccountBucketUtil.s3.send(new Operation(params)); } else { const command = new Operation(params); - + // Create unsigned client const unsignedClient = new BucketUtility('default', { ...sigCfg, @@ -68,9 +68,9 @@ withV4(sigCfg => { step: 'serialize', priority: 'high', before: 'awsAuthMiddleware', - } + }, ); - } + }, }); return unsignedClient.s3.send(command); } @@ -102,8 +102,7 @@ withV4(sigCfg => { const grantUri = `uri=${auth ? constants.allAuthedUsersId : constants.publicId}`; // TODO fix flakiness on E2E and re-enable, see CLDSRV-254 - describeSkipIfE2E('PUT Bucket ACL using predefined groups - ' + - `${authType} request`, () => { + describeSkipIfE2E('PUT Bucket ACL using predefined groups - ' + `${authType} request`, () => { const aclParam = { Bucket: testBucket, ACL: 'private', @@ -111,11 +110,13 @@ withV4(sigCfg => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: testBucket })); - await s3.send(new PutObjectCommand({ - Bucket: testBucket, - Body: testBody, - Key: ownerObjKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Body: testBody, + Key: ownerObjKey, + }), + ); }); afterEach(async () => { @@ -123,16 +124,24 @@ withV4(sigCfg => { await ownerAccountBucketUtil.deleteOne(testBucket); }); - it('should grant read access', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantRead: grantUri, - })) + it('should grant read access', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantRead: grantUri, + }), + ) .then(() => awsRequest(auth, ListObjectsV2Command, { Bucket: testBucket }))); - it('should grant read access with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) + it('should grant read access with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) .then(() => awsRequest(auth, ListObjectsV2Command, { Bucket: testBucket }))); it('should not grant read access', done => { @@ -143,57 +152,81 @@ withV4(sigCfg => { // Don't return the promise! }); - it('should grant write access', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWrite: grantUri, - })) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: testKey, - }))); - - it('should grant write access with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: testKey, - }))); + it('should grant write access', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantWrite: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + )); + + it('should grant write access with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + )); it('should not grant write access', done => { s3.send(new PutBucketAclCommand(aclParam)) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: testKey, - })) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); - itSkipIfE2E('should grant write access on an object not owned by the grantee', - () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWrite: grantUri, - })) - .then(() => awsRequest(auth, PutObjectCommand, { - Bucket: testBucket, - Body: testBody, - Key: ownerObjKey, - }))); + itSkipIfE2E('should grant write access on an object not owned by the grantee', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantWrite: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutObjectCommand, { + Bucket: testBucket, + Body: testBody, + Key: ownerObjKey, + }), + ), + ); it(`should ${auth ? '' : 'not '}delete object not owned by the grantee`, done => { - s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWrite: grantUri, - })) - .then(() => awsRequest(auth, DeleteObjectCommand, { + s3.send( + new PutBucketAclCommand({ Bucket: testBucket, - Key: ownerObjKey, - })) + GrantWrite: grantUri, + }), + ) + .then(() => + awsRequest(auth, DeleteObjectCommand, { + Bucket: testBucket, + Key: ownerObjKey, + }), + ) .then(() => { if (auth) { done(); @@ -210,16 +243,24 @@ withV4(sigCfg => { }); }); - it('should read bucket acl', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantReadACP: grantUri, - })) + it('should read bucket acl', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantReadACP: grantUri, + }), + ) .then(() => awsRequest(auth, GetBucketAclCommand, { Bucket: testBucket }))); - it('should read bucket acl with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) + it('should read bucket acl with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) .then(() => awsRequest(auth, GetBucketAclCommand, { Bucket: testBucket }))); it('should not read bucket acl', done => { @@ -229,30 +270,44 @@ withV4(sigCfg => { .catch(cbWithError(done)); }); - it('should write bucket acl', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantWriteACP: grantUri, - })) - .then(() => awsRequest(auth, PutBucketAclCommand, { - Bucket: testBucket, - GrantReadACP: `uri=${constants.publicId}`, - }))); - - it('should write bucket acl with grant-full-control', () => s3.send(new PutBucketAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - })) - .then(() => awsRequest(auth, PutBucketAclCommand, { - Bucket: testBucket, - GrantReadACP: `uri=${constants.publicId}`, - }))); + it('should write bucket acl', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantWriteACP: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutBucketAclCommand, { + Bucket: testBucket, + GrantReadACP: `uri=${constants.publicId}`, + }), + )); + + it('should write bucket acl with grant-full-control', () => + s3 + .send( + new PutBucketAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + }), + ) + .then(() => + awsRequest(auth, PutBucketAclCommand, { + Bucket: testBucket, + GrantReadACP: `uri=${constants.publicId}`, + }), + )); it('should not write bucket acl', done => { s3.send(new PutBucketAclCommand(aclParam)) - .then(() => awsRequest(auth, PutBucketAclCommand, { - Bucket: testBucket, - GrantReadACP: `uri=${constants.allAuthedUsersId}`, - })) + .then(() => + awsRequest(auth, PutBucketAclCommand, { + Bucket: testBucket, + GrantReadACP: `uri=${constants.allAuthedUsersId}`, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); @@ -267,11 +322,13 @@ withV4(sigCfg => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: testBucket })); - await s3.send(new PutObjectCommand({ - Bucket: testBucket, - Body: testBody, - Key: testKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Body: testBody, + Key: testKey, + }), + ); }); afterEach(async () => { @@ -279,95 +336,137 @@ withV4(sigCfg => { await ownerAccountBucketUtil.deleteOne(testBucket); }); - it('should grant read access', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantRead: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectCommand, { - Bucket: testBucket, - Key: testKey, - }))); - - it('should grant read access with grant-full-control', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectCommand, { - Bucket: testBucket, - Key: testKey, - }))); + it('should grant read access', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantRead: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); + + it('should grant read access with grant-full-control', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); it('should not grant read access', done => { s3.send(new PutObjectAclCommand(aclParam)) - .then(() => awsRequest(auth, GetObjectCommand, { - Bucket: testBucket, - Key: testKey, - })) + .then(() => + awsRequest(auth, GetObjectCommand, { + Bucket: testBucket, + Key: testKey, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); - it('should read object acl', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantReadACP: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - }))); - - it('should read object acl with grant-full-control', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, GetObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - }))); + it('should read object acl', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantReadACP: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); + + it('should read object acl with grant-full-control', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, GetObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + }), + )); it('should not read object acl', done => { s3.send(new PutObjectAclCommand(aclParam)) - .then(() => awsRequest(auth, GetObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - })) + .then(() => + awsRequest(auth, GetObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); - it('should write object acl', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantWriteACP: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, PutObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - GrantReadACP: grantUri, - }))); - - it('should write object acl with grant-full-control', () => s3.send(new PutObjectAclCommand({ - Bucket: testBucket, - GrantFullControl: grantUri, - Key: testKey, - })) - .then(() => awsRequest(auth, PutObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - GrantReadACP: `uri=${constants.publicId}`, - }))); + it('should write object acl', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantWriteACP: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, PutObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + GrantReadACP: grantUri, + }), + )); + + it('should write object acl with grant-full-control', () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: testBucket, + GrantFullControl: grantUri, + Key: testKey, + }), + ) + .then(() => + awsRequest(auth, PutObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + GrantReadACP: `uri=${constants.publicId}`, + }), + )); it('should not write object acl', done => { s3.send(new PutObjectAclCommand(aclParam)) - .then(() => awsRequest(auth, PutObjectAclCommand, { - Bucket: testBucket, - Key: testKey, - GrantReadACP: `uri=${constants.allAuthedUsersId}`, - })) + .then(() => + awsRequest(auth, PutObjectAclCommand, { + Bucket: testBucket, + Key: testKey, + GrantReadACP: `uri=${constants.allAuthedUsersId}`, + }), + ) .then(() => done(new Error('Expected failure'))) .catch(cbWithError(done)); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js index cea6eff071..38b72c782f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js +++ b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyBypassPort.js @@ -1,15 +1,15 @@ const assert = require('assert'); -const { - S3Client, - PutObjectCommand, +const { + S3Client, + PutObjectCommand, GetObjectCommand, PutBucketPolicyCommand, DeleteBucketPolicyCommand, } = require('@aws-sdk/client-s3'); -const { - IAMClient, - CreatePolicyCommand, - CreateUserCommand, +const { + IAMClient, + CreatePolicyCommand, + CreateUserCommand, AttachUserPolicyCommand, DetachUserPolicyCommand, DeletePolicyCommand, @@ -18,12 +18,9 @@ const { CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, - DeleteRoleCommand + DeleteRoleCommand, } = require('@aws-sdk/client-iam'); -const { - STSClient, - AssumeRoleCommand -} = require('@aws-sdk/client-sts'); +const { STSClient, AssumeRoleCommand } = require('@aws-sdk/client-sts'); const { v4: uuid } = require('uuid'); const getConfig = require('../support/config'); @@ -55,50 +52,58 @@ describeBypass('Bucket Policy Bypass Port', () => { before(async () => { await bucketUtilAccount.createOne(bucketName); - await s3ClientAccount.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Body: objectContent, - })); + await s3ClientAccount.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Body: objectContent, + }), + ); - await s3ClientAccount.send(new PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Sid: 'DenyAllAccess', - Effect: 'Deny', - Principal: '*', - Action: 's3:*', - Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], - }, - ], + await s3ClientAccount.send( + new PutBucketPolicyCommand({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'DenyAllAccess', + Effect: 'Deny', + Principal: '*', + Action: 's3:*', + Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], + }, + ], + }), }), - })); + ); // create iam policy allow all actions for user and role - const policyRes = await iamClient.send(new CreatePolicyCommand({ - PolicyName: 'bp-bypass-policy', - PolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Sid: 'AllowAllActions', - Effect: 'Allow', - Action: '*', - Resource: ['*'], - }, - ], + const policyRes = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: 'bp-bypass-policy', + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'AllowAllActions', + Effect: 'Allow', + Action: '*', + Resource: ['*'], + }, + ], + }), }), - })); + ); policyAllowAllActions = policyRes.Policy; await iamClient.send(new CreateUserCommand({ UserName: userName })); - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); }); after(async () => { @@ -108,20 +113,24 @@ describeBypass('Bucket Policy Bypass Port', () => { await bucketUtilAccount.deleteOne(bucketName); if (policyAllowAllActions) { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyAllowAllActions.Arn })); } await iamClient.send(new DeleteUserCommand({ UserName: userName })); }); it('should allow account root access on s3 port', async () => { - const getResponse = await s3ClientAccount.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const getResponse = await s3ClientAccount.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert(getResponse.Body, 'Should be able to get object'); const bodyString = await getResponse.Body.transformToString(); assert.strictEqual(bodyString, objectContent); @@ -157,10 +166,12 @@ describeBypass('Bucket Policy Bypass Port', () => { it('should deny user access on s3 port', async () => { try { - await userS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + await userS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert.fail('Expected AccessDenied error for getObject'); } catch (err) { assert.strictEqual(err.name, 'AccessDenied'); @@ -168,10 +179,12 @@ describeBypass('Bucket Policy Bypass Port', () => { }); it('should bypass user bucket policy on internal port', async () => { - const getResponse = await userInternalBypassBPS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const getResponse = await userInternalBypassBPS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert(getResponse.Body, 'Should be able to get object on internal port'); const bodyString = await getResponse.Body.transformToString(); assert.strictEqual(bodyString, objectContent); @@ -184,24 +197,28 @@ describeBypass('Bucket Policy Bypass Port', () => { let stsClient; before(async () => { - const roleRes = await iamClient.send(new CreateRoleCommand({ - RoleName: roleName, - AssumeRolePolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Effect: 'Allow', - Principal: '*', - Action: 'sts:AssumeRole', - }, - ], + const roleRes = await iamClient.send( + new CreateRoleCommand({ + RoleName: roleName, + AssumeRolePolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: '*', + Action: 'sts:AssumeRole', + }, + ], + }), }), - })); + ); - await iamClient.send(new AttachRolePolicyCommand({ - RoleName: roleName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new AttachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); const accessKeyResponse = await iamClient.send(new CreateAccessKeyCommand({ UserName: userName })); @@ -216,10 +233,12 @@ describeBypass('Bucket Policy Bypass Port', () => { stsClient = new STSClient(stsConfig); // Assume role to get temporary credentials - const assumeRoleResponse = await stsClient.send(new AssumeRoleCommand({ - RoleArn: roleRes.Role.Arn, - RoleSessionName: 'bp-bypass-session', - })); + const assumeRoleResponse = await stsClient.send( + new AssumeRoleCommand({ + RoleArn: roleRes.Role.Arn, + RoleSessionName: 'bp-bypass-session', + }), + ); const credentials = assumeRoleResponse.Credentials; // Create S3 client for role (regular port) @@ -245,19 +264,23 @@ describeBypass('Bucket Policy Bypass Port', () => { }); after(async () => { - await iamClient.send(new DetachRolePolicyCommand({ - RoleName: roleName, - PolicyArn: policyAllowAllActions.Arn, - })); + await iamClient.send( + new DetachRolePolicyCommand({ + RoleName: roleName, + PolicyArn: policyAllowAllActions.Arn, + }), + ); await iamClient.send(new DeleteRoleCommand({ RoleName: roleName })); }); it('should deny role access on s3 port', async () => { try { - await roleS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + await roleS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert.fail('Expected AccessDenied error for getObject'); } catch (err) { assert.strictEqual(err.name, 'AccessDenied'); @@ -265,10 +288,12 @@ describeBypass('Bucket Policy Bypass Port', () => { }); it('should bypass role bucket policy on internal port', async () => { - const getResponse = await roleInternalBypassBPS3Client.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const getResponse = await roleInternalBypassBPS3Client.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); assert(getResponse.Body, 'Should be able to get object on internal port'); const bodyString = await getResponse.Body.transformToString(); assert.strictEqual(bodyString, objectContent); diff --git a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js index b45d1e4209..984e54138f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js +++ b/tests/functional/aws-node-sdk/test/bucket/bucketPolicyWithResourceStatements.js @@ -3,7 +3,8 @@ const { PutBucketPolicyCommand, ListObjectsCommand, GetObjectCommand, - PutObjectCommand } = require('@aws-sdk/client-s3'); + PutObjectCommand, +} = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); const withV4 = require('../support/withV4'); @@ -18,12 +19,13 @@ withV4(sigCfg => { if (auth) { // Use authenticated client const commandMap = { - 'listObjects': ListObjectsCommand, - 'getObject': GetObjectCommand, - 'putObject': PutObjectCommand, + listObjects: ListObjectsCommand, + getObject: GetObjectCommand, + putObject: PutObjectCommand, }; const CommandCtor = commandMap[operation]; - ownerAccountBucketUtil.s3.send(new CommandCtor(params)) + ownerAccountBucketUtil.s3 + .send(new CommandCtor(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } else { @@ -35,12 +37,13 @@ withV4(sigCfg => { signer: { sign: async request => request }, }); const commandMap = { - 'listObjects': ListObjectsCommand, - 'getObject': GetObjectCommand, - 'putObject': PutObjectCommand, + listObjects: ListObjectsCommand, + getObject: GetObjectCommand, + putObject: PutObjectCommand, }; const CommandCtor = commandMap[operation]; - unauthClient.s3.send(new CommandCtor(params)) + unauthClient.s3 + .send(new CommandCtor(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } @@ -62,8 +65,9 @@ withV4(sigCfg => { describe('Bucket policies with resource statement', () => { beforeEach(() => ownerAccountBucketUtil.createMany(testBuckets)); - afterEach(() => ownerAccountBucketUtil.emptyMany(testBuckets) - .then(() => ownerAccountBucketUtil.deleteMany(testBuckets))); + afterEach(() => + ownerAccountBucketUtil.emptyMany(testBuckets).then(() => ownerAccountBucketUtil.deleteMany(testBuckets)), + ); it('should allow action on a bucket specified in the policy', done => { const statement = { @@ -77,11 +81,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[0] }; awsRequest(true, 'listObjects', param, cbNoError(done)); }); @@ -99,11 +104,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[1] }; awsRequest(false, 'listObjects', param, cbWithError(done)); }); @@ -121,11 +127,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[0] }; awsRequest(false, 'listObjects', param, cbWithError(done)); }); @@ -145,22 +152,28 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => s3.send(new PutObjectCommand({ - Bucket: testBuckets[0], - Body: testBody, - Key: testKey, - }))) - .then(() => { - const param = { + s3.send( + new PutBucketPolicyCommand({ Bucket: testBuckets[0], - Key: testKey, - }; - awsRequest(false, 'getObject', param, cbNoError(done)); - }); + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: testBuckets[0], + Body: testBody, + Key: testKey, + }), + ), + ) + .then(() => { + const param = { + Bucket: testBuckets[0], + Key: testKey, + }; + awsRequest(false, 'getObject', param, cbNoError(done)); + }); }); it('should allow action on an object satisfying the wildcard in the policy', done => { @@ -177,22 +190,28 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => s3.send(new PutObjectCommand({ - Bucket: testBuckets[0], - Body: testBody, - Key: testKey, - }))) - .then(() => { - const param = { + s3.send( + new PutBucketPolicyCommand({ Bucket: testBuckets[0], - Key: testKey, - }; - awsRequest(false, 'getObject', param, cbNoError(done)); - }); + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: testBuckets[0], + Body: testBody, + Key: testKey, + }), + ), + ) + .then(() => { + const param = { + Bucket: testBuckets[0], + Key: testKey, + }; + awsRequest(false, 'getObject', param, cbNoError(done)); + }); }); it('should deny action on an object specified in the policy', done => { @@ -209,22 +228,28 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => s3.send(new PutObjectCommand({ + s3.send( + new PutBucketPolicyCommand({ Bucket: testBuckets[0], - Body: testBody, - Key: testKey, - }))) - .then(() => { - const param = { - Bucket: testBuckets[0], - Key: testKey, - }; - awsRequest(false, 'getObject', param, cbWithError(done)); - }); + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: testBuckets[0], + Body: testBody, + Key: testKey, + }), + ), + ) + .then(() => { + const param = { + Bucket: testBuckets[0], + Key: testKey, + }; + awsRequest(false, 'getObject', param, cbWithError(done)); + }); }); it('should deny action on an object not specified in the policy', done => { @@ -240,11 +265,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[0], Key: 'invalidkey', @@ -266,11 +292,12 @@ withV4(sigCfg => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBuckets[0], - Policy: JSON.stringify(bucketPolicy), - })) - .then(() => { + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBuckets[0], + Policy: JSON.stringify(bucketPolicy), + }), + ).then(() => { const param = { Bucket: testBuckets[1], Key: 'invalidkey', diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js index d454c15752..6f5e8ebd40 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketLifecycle.js @@ -1,11 +1,13 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketLifecycleCommand, PutBucketLifecycleConfigurationCommand, - GetBucketLifecycleConfigurationCommand } = require('@aws-sdk/client-s3'); + GetBucketLifecycleConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); @@ -24,11 +26,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.Code}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.Code}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -67,12 +74,11 @@ describe('aws-sdk test delete bucket lifecycle', () => { } }); - it('should return no error if no lifecycle config on bucket', () => s3.send(new - DeleteBucketLifecycleCommand({ Bucket: bucket }))); + it('should return no error if no lifecycle config on bucket', () => + s3.send(new DeleteBucketLifecycleCommand({ Bucket: bucket }))); it('should delete lifecycle configuration from bucket', async () => { - const params = { Bucket: bucket, - LifecycleConfiguration: { Rules: [basicRule] } }; + const params = { Bucket: bucket, LifecycleConfiguration: { Rules: [basicRule] } }; await s3.send(new PutBucketLifecycleConfigurationCommand(params)); await s3.send(new DeleteBucketLifecycleCommand({ Bucket: bucket })); try { diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js index 3276283c3d..97ad6c4e89 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketPolicy.js @@ -1,11 +1,13 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketPolicyCommand, PutBucketPolicyCommand, - GetBucketPolicyCommand } = require('@aws-sdk/client-s3'); + GetBucketPolicyCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -13,13 +15,15 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'deletebucketpolicy-test-bucket'; const bucketPolicy = { Version: '2012-10-17', - Statement: [{ - Sid: 'testid', - Effect: 'Allow', - Principal: '*', - Action: 's3:putBucketPolicy', - Resource: `arn:aws:s3:::${bucket}`, - }], + Statement: [ + { + Sid: 'testid', + Effect: 'Allow', + Principal: '*', + Action: 's3:putBucketPolicy', + Resource: `arn:aws:s3:::${bucket}`, + }, + ], }; // Check for the expected error response code and status code. @@ -27,11 +31,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js index 0e58a7b901..e1f1aa723b 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketQuota.js @@ -1,6 +1,4 @@ -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); const sendRequest = require('../quota/tooling').sendRequest; diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js index d232f6d206..3cde37edde 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketRateLimit.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - S3Client, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const { sendRateLimitRequest, skipIfRateLimitDisabled } = require('../rateLimit/tooling'); @@ -36,17 +32,19 @@ skipIfRateLimitDisabled('Test delete bucket rate limit', () => { try { // First set a rate limit config const rateLimitConfig = { RequestsPerSecond: 150 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); // Then delete it - await sendRateLimitRequest('DELETE', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?rate-limit`); // Verify it's deleted try { - await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.fail('Expected NoSuchRateLimitConfig error'); } catch (err) { assert.strictEqual(err.Error.Code[0], 'NoSuchRateLimitConfig'); @@ -58,8 +56,7 @@ skipIfRateLimitDisabled('Test delete bucket rate limit', () => { it('should not return an error even if no rate limit config exists', async () => { try { - await sendRateLimitRequest('DELETE', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest('DELETE', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.ok(true); } catch (err) { assert.ifError(err); @@ -68,8 +65,7 @@ skipIfRateLimitDisabled('Test delete bucket rate limit', () => { it('should return NoSuchBucket error when bucket does not exist', async () => { try { - await sendRateLimitRequest('DELETE', '127.0.0.1:8000', - `/${nonExistentBucket}/?rate-limit`); + await sendRateLimitRequest('DELETE', '127.0.0.1:8000', `/${nonExistentBucket}/?rate-limit`); } catch (err) { assert.strictEqual(err.Error.Code[0], 'NoSuchBucket'); } diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js index ee3212aa04..a4bea67e04 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketReplication.js @@ -1,11 +1,13 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand, - GetBucketReplicationCommand } = require('@aws-sdk/client-s3'); + GetBucketReplicationCommand, +} = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); const getConfig = require('../support/config'); @@ -13,8 +15,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'source-bucket'; const replicationConfig = { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', Rules: [ { Destination: { Bucket: 'arn:aws:s3:::destination-bucket' }, @@ -31,24 +32,27 @@ describe('aws-node-sdk test deleteBucketReplication', () => { const config = getConfig('default', { signatureVersion: 'v4' }); function putVersioningOnBucket(bucket) { - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })); + return s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); } function putReplicationOnBucket(bucket) { - return s3.send(new PutBucketReplicationCommand({ - Bucket: bucket, - ReplicationConfiguration: replicationConfig, - })); + return s3.send( + new PutBucketReplicationCommand({ + Bucket: bucket, + ReplicationConfiguration: replicationConfig, + }), + ); } function deleteReplicationAndCheckResponse(bucket) { - return s3.send(new DeleteBucketReplicationCommand({ Bucket: bucket })) - .then(data => { - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - }); + return s3.send(new DeleteBucketReplicationCommand({ Bucket: bucket })).then(data => { + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + }); } beforeEach(() => { @@ -59,8 +63,8 @@ describe('aws-node-sdk test deleteBucketReplication', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - it('should return empty object if bucket has no replication config', - () => deleteReplicationAndCheckResponse(bucket)); + it('should return empty object if bucket has no replication config', () => + deleteReplicationAndCheckResponse(bucket)); it('should delete a bucket replication config when it has one', async () => { await putVersioningOnBucket(bucket); @@ -68,23 +72,26 @@ describe('aws-node-sdk test deleteBucketReplication', () => { await deleteReplicationAndCheckResponse(bucket); }); - it('should return ReplicationConfigurationNotFoundError if getting ' + - 'replication config after it has been deleted', async () => { - await putVersioningOnBucket(bucket); - await putReplicationOnBucket(bucket); - - const data = await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); - assert.deepStrictEqual(data.ReplicationConfiguration, replicationConfig); + it( + 'should return ReplicationConfigurationNotFoundError if getting ' + + 'replication config after it has been deleted', + async () => { + await putVersioningOnBucket(bucket); + await putReplicationOnBucket(bucket); - await deleteReplicationAndCheckResponse(bucket); - - try { - await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); - assert.fail('Expected ReplicationConfigurationNotFoundError'); - } catch (err) { - assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.name]); - } - }); + const data = await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); + assert.deepStrictEqual(data.ReplicationConfiguration, replicationConfig); + + await deleteReplicationAndCheckResponse(bucket); + + try { + await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); + assert.fail('Expected ReplicationConfigurationNotFoundError'); + } catch (err) { + assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.name]); + } + }, + ); it('should return AccessDenied if user is not bucket owner', async () => { try { diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js b/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js index 2eecf0dd7a..e770e4b1a8 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteBucketTagging.js @@ -1,10 +1,12 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketTaggingCommand, GetBucketTaggingCommand, - DeleteBucketTaggingCommand } = require('@aws-sdk/client-s3'); + DeleteBucketTaggingCommand, +} = require('@aws-sdk/client-s3'); const assertError = require('../../../../utilities/bucketTagging-util'); @@ -39,25 +41,33 @@ describe('aws-sdk test delete bucket tagging', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); it('should delete tag', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validTagging, - Bucket: bucket, - })); - const res = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validTagging, + Bucket: bucket, + }), + ); + const res = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); assert.deepStrictEqual(res.TagSet, validTagging.TagSet); - await s3.send(new DeleteBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); - try { - await s3.send(new GetBucketTaggingCommand({ + await s3.send( + new DeleteBucketTaggingCommand({ AccountId: s3.AccountId, Bucket: bucket, - })); + }), + ); + try { + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -66,23 +76,29 @@ describe('aws-sdk test delete bucket tagging', () => { it('should make no change when deleting tags on bucket with no tags', async () => { try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); } - await s3.send(new DeleteBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); - try { - await s3.send(new GetBucketTaggingCommand({ + await s3.send( + new DeleteBucketTaggingCommand({ AccountId: s3.AccountId, Bucket: bucket, - })); + }), + ); + try { + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteCors.js b/tests/functional/aws-node-sdk/test/bucket/deleteCors.js index 88196a5276..647d4d429b 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteCors.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteCors.js @@ -1,26 +1,29 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketCorsCommand, PutBucketCorsCommand, - GetBucketCorsCommand } = require('@aws-sdk/client-s3'); + GetBucketCorsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); const bucketName = 'testdeletecorsbucket'; -const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000, - ExposeHeaders: ['x-amz-server-side-encryption'] }, - { AllowedMethods: ['GET'], - AllowedOrigins: ['*'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000 }, -] }; +const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: ['*'], + MaxAgeSeconds: 3000, + ExposeHeaders: ['x-amz-server-side-encryption'], + }, + { AllowedMethods: ['GET'], AllowedOrigins: ['*'], AllowedHeaders: ['*'], MaxAgeSeconds: 3000 }, + ], +}; const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; @@ -49,30 +52,31 @@ describe('DELETE bucket cors', () => { describe('with existing bucket', () => { beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); - + afterEach(() => deleteBucket(s3, bucketName)); describe('without existing cors configuration', () => { it('should return a 204 response', async () => { const res = await s3.send(new DeleteBucketCorsCommand({ Bucket: bucketName })); const statusCode = res?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, 204, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, 204, `Found unexpected statusCode ${statusCode}`); }); }); describe('with existing cors configuration', () => { - beforeEach(() => s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors - }))); - + beforeEach(() => + s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ), + ); it('should delete bucket configuration successfully', async () => { const res = await s3.send(new DeleteBucketCorsCommand({ Bucket: bucketName })); const statusCode = res?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, 204, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, 204, `Found unexpected statusCode ${statusCode}`); try { await s3.send(new GetBucketCorsCommand({ Bucket: bucketName })); throw new Error('Expected NoSuchCORSConfiguration error'); @@ -88,8 +92,7 @@ describe('DELETE bucket cors', () => { // to add a second set of real aws credentials under a profile // named 'lisa' in ~/.aws/scality, then rename 'itSkipIfAWS' to // 'it'. - itSkipIfAWS('should return AccessDenied if user is not bucket' + - 'owner', async () => { + itSkipIfAWS('should return AccessDenied if user is not bucket' + 'owner', async () => { try { await otherAccountS3.send(new DeleteBucketCorsCommand({ Bucket: bucketName })); throw new Error('Expected AccessDenied error'); diff --git a/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js b/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js index 7518375084..aed1070f8a 100644 --- a/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js +++ b/tests/functional/aws-node-sdk/test/bucket/deleteWebsite.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, DeleteBucketWebsiteCommand, - PutBucketWebsiteCommand } = require('@aws-sdk/client-s3'); + PutBucketWebsiteCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); @@ -40,22 +42,23 @@ describe('DELETE bucket website', () => { it('should return a 204 response', async () => { const res = await s3.send(new DeleteBucketWebsiteCommand({ Bucket: bucketName })); const statusCode = res?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, 204, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, 204, `Found unexpected statusCode ${statusCode}`); }); }); describe('with existing configuration', () => { beforeEach(() => { const config = new WebsiteConfigTester('index.html'); - return s3.send(new PutBucketWebsiteCommand({ - Bucket: bucketName, - WebsiteConfiguration: config - })); + return s3.send( + new PutBucketWebsiteCommand({ + Bucket: bucketName, + WebsiteConfiguration: config, + }), + ); }); - it('should delete bucket configuration successfully', () => s3.send(new - DeleteBucketWebsiteCommand({ Bucket: bucketName }))); + it('should delete bucket configuration successfully', () => + s3.send(new DeleteBucketWebsiteCommand({ Bucket: bucketName }))); it('should return AccessDenied if user is not bucket owner', async () => { try { diff --git a/tests/functional/aws-node-sdk/test/bucket/get.js b/tests/functional/aws-node-sdk/test/bucket/get.js index 43f0602cca..5b0bf616b5 100644 --- a/tests/functional/aws-node-sdk/test/bucket/get.js +++ b/tests/functional/aws-node-sdk/test/bucket/get.js @@ -34,276 +34,240 @@ const vaultHost = config.vaultd?.host || 'localhost'; const tests = [ { name: 'return created objects in alphabetical order', - objectPutParams: Bucket => - [ - { Bucket, Key: 'testB/' }, - { Bucket, Key: 'testB/test.json', Body: '{}' }, - { Bucket, Key: 'testA/' }, - { Bucket, Key: 'testA/test.json', Body: '{}' }, - { Bucket, Key: 'testA/test/test.json', Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'testB/' }, + { Bucket, Key: 'testB/test.json', Body: '{}' }, + { Bucket, Key: 'testA/' }, + { Bucket, Key: 'testA/test.json', Body: '{}' }, + { Bucket, Key: 'testA/test/test.json', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket }), assertions: (data, Bucket) => { const keys = data.Contents.map(object => object.Key); // ETag should include quotes around value - const emptyObjectHash = - '"d41d8cd98f00b204e9800998ecf8427e"'; + const emptyObjectHash = '"d41d8cd98f00b204e9800998ecf8427e"'; assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(keys, [ - 'testA/', - 'testA/test.json', - 'testA/test/test.json', - 'testB/', - 'testB/test.json', - ], 'Bucket content mismatch'); - assert.deepStrictEqual(data.Contents[0].ETag, - emptyObjectHash, 'Object hash mismatch'); + assert.deepEqual( + keys, + ['testA/', 'testA/test.json', 'testA/test/test.json', 'testB/', 'testB/test.json'], + 'Bucket content mismatch', + ); + assert.deepStrictEqual(data.Contents[0].ETag, emptyObjectHash, 'Object hash mismatch'); }, }, { name: 'return multiple common prefixes', - objectPutParams: Bucket => - [ - { Bucket, Key: 'testB/' }, - { Bucket, Key: 'testB/test.json', Body: '{}' }, - { Bucket, Key: 'testA/' }, - { Bucket, Key: 'testA/test.json', Body: '{}' }, - { Bucket, Key: 'testA/test/test.json', Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'testB/' }, + { Bucket, Key: 'testB/test.json', Body: '{}' }, + { Bucket, Key: 'testA/' }, + { Bucket, Key: 'testA/test.json', Body: '{}' }, + { Bucket, Key: 'testA/test/test.json', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket, Delimiter: '/' }), assertions: (data, Bucket) => { const prefixes = data.CommonPrefixes.map(cp => cp.Prefix); assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(prefixes, [ - 'testA/', - 'testB/', - ], 'Bucket content mismatch'); + assert.deepEqual(prefixes, ['testA/', 'testB/'], 'Bucket content mismatch'); }, }, { name: 'list objects with percentage delimiter', - objectPutParams: Bucket => - [ - { Bucket, Key: 'testB%' }, - { Bucket, Key: 'testC%test.json', Body: '{}' }, - { Bucket, Key: 'testA%' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'testB%' }, + { Bucket, Key: 'testC%test.json', Body: '{}' }, + { Bucket, Key: 'testA%' }, + ], listObjectParams: Bucket => ({ Bucket, Delimiter: '%' }), assertions: data => { const prefixes = data.CommonPrefixes.map(cp => cp.Prefix); - assert.deepEqual(prefixes, [ - 'testA%', - 'testB%', - 'testC%', - ], 'Bucket content mismatch'); + assert.deepEqual(prefixes, ['testA%', 'testB%', 'testC%'], 'Bucket content mismatch'); }, }, { name: 'list object titles with white spaces', - objectPutParams: Bucket => - [ - { Bucket, Key: 'whiteSpace/' }, - { Bucket, Key: 'whiteSpace/one whiteSpace', Body: '{}' }, - { Bucket, Key: 'whiteSpace/two white spaces', Body: '{}' }, - { Bucket, Key: 'white space/' }, - { Bucket, Key: 'white space/one whiteSpace', Body: '{}' }, - { Bucket, Key: 'white space/two white spaces', Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'whiteSpace/' }, + { Bucket, Key: 'whiteSpace/one whiteSpace', Body: '{}' }, + { Bucket, Key: 'whiteSpace/two white spaces', Body: '{}' }, + { Bucket, Key: 'white space/' }, + { Bucket, Key: 'white space/one whiteSpace', Body: '{}' }, + { Bucket, Key: 'white space/two white spaces', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket }), assertions: (data, Bucket) => { const keys = data.Contents.map(object => object.Key); assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(keys, [ - /* These object names are intentionally listed in a + assert.deepEqual( + keys, + [ + /* These object names are intentionally listed in a different order than they were created to additionally test that they are listed alphabetically. */ - 'white space/', - 'white space/one whiteSpace', - 'white space/two white spaces', - 'whiteSpace/', - 'whiteSpace/one whiteSpace', - 'whiteSpace/two white spaces', - ], 'Bucket content mismatch'); + 'white space/', + 'white space/one whiteSpace', + 'white space/two white spaces', + 'whiteSpace/', + 'whiteSpace/one whiteSpace', + 'whiteSpace/two white spaces', + ], + 'Bucket content mismatch', + ); }, }, { name: 'list object titles that contain special chars', - objectPutParams: Bucket => - [ - { Bucket, Key: 'foo&<>\'"' }, - { Bucket, Key: '*asterixObjTitle/' }, - { Bucket, Key: '*asterixObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '*asterixObjTitle/*asterixObjTitle', - Body: '{}' }, - { Bucket, Key: '.dotObjTitle/' }, - { Bucket, Key: '.dotObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '.dotObjTitle/.dotObjTitle', Body: '{}' }, - { Bucket, Key: '(openParenObjTitle/' }, - { Bucket, Key: '(openParenObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '(openParenObjTitle/(openParenObjTitle', - Body: '{}' }, - { Bucket, Key: ')closeParenObjTitle/' }, - { Bucket, Key: ')closeParenObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: ')closeParenObjTitle/)closeParenObjTitle', - Body: '{}' }, - { Bucket, Key: '!exclamationPointObjTitle/' }, - { Bucket, Key: '!exclamationPointObjTitle/objTitleA', - Body: '{}' }, - { Bucket, Key: - '!exclamationPointObjTitle/!exclamationPointObjTitle', - Body: '{}' }, - { Bucket, Key: '-dashObjTitle/' }, - { Bucket, Key: '-dashObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '-dashObjTitle/-dashObjTitle', Body: '{}' }, - { Bucket, Key: '_underscoreObjTitle/' }, - { Bucket, Key: '_underscoreObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: '_underscoreObjTitle/_underscoreObjTitle', - Body: '{}' }, - { Bucket, Key: "'apostropheObjTitle/" }, - { Bucket, Key: "'apostropheObjTitle/objTitleA", Body: '{}' }, - { Bucket, Key: "'apostropheObjTitle/'apostropheObjTitle", - Body: '{}' }, - { Bucket, Key: 'çcedilleObjTitle' }, - { Bucket, Key: 'çcedilleObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'çcedilleObjTitle/çcedilleObjTitle', - Body: '{}' }, - { Bucket, Key: 'дcyrillicDObjTitle' }, - { Bucket, Key: 'дcyrillicDObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'дcyrillicDObjTitle/дcyrillicDObjTitle', - Body: '{}' }, - { Bucket, Key: 'ñenyeObjTitle' }, - { Bucket, Key: 'ñenyeObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'ñenyeObjTitle/ñenyeObjTitle', Body: '{}' }, - { Bucket, Key: '山chineseMountainObjTitle' }, - { Bucket, Key: '山chineseMountainObjTitle/objTitleA', - Body: '{}' }, - { Bucket, Key: - '山chineseMountainObjTitle/山chineseMountainObjTitle', - Body: '{}' }, - { Bucket, Key: 'àaGraveLowerCaseObjTitle' }, - { Bucket, Key: 'àaGraveLowerCaseObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', - Body: '{}' }, - { Bucket, Key: 'ÀaGraveUpperCaseObjTitle' }, - { Bucket, Key: 'ÀaGraveUpperCaseObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', - Body: '{}' }, - { Bucket, Key: 'ßscharfesSObjTitle' }, - { Bucket, Key: 'ßscharfesSObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'ßscharfesSObjTitle/ßscharfesSObjTitle', - Body: '{}' }, - { Bucket, Key: '日japaneseMountainObjTitle' }, - { Bucket, Key: '日japaneseMountainObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: '日japaneseMountainObjTitle/日japaneseMountainObjTitle', - Body: '{}' }, - { Bucket, Key: 'بbaArabicObjTitle' }, - { Bucket, Key: 'بbaArabicObjTitle/objTitleA', Body: '{}' }, - { Bucket, Key: 'بbaArabicObjTitle/بbaArabicObjTitle', - Body: '{}' }, - { Bucket, - Key: 'अadevanagariHindiObjTitle' }, - { Bucket, - Key: 'अadevanagariHindiObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', - Body: '{}' }, - { Bucket, Key: 'éeacuteLowerCaseObjTitle' }, - { Bucket, Key: 'éeacuteLowerCaseObjTitle/objTitleA', - Body: '{}' }, - { Bucket, - Key: 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', - Body: '{}' }, - ], + objectPutParams: Bucket => [ + { Bucket, Key: 'foo&<>\'"' }, + { Bucket, Key: '*asterixObjTitle/' }, + { Bucket, Key: '*asterixObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '*asterixObjTitle/*asterixObjTitle', Body: '{}' }, + { Bucket, Key: '.dotObjTitle/' }, + { Bucket, Key: '.dotObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '.dotObjTitle/.dotObjTitle', Body: '{}' }, + { Bucket, Key: '(openParenObjTitle/' }, + { Bucket, Key: '(openParenObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '(openParenObjTitle/(openParenObjTitle', Body: '{}' }, + { Bucket, Key: ')closeParenObjTitle/' }, + { Bucket, Key: ')closeParenObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: ')closeParenObjTitle/)closeParenObjTitle', Body: '{}' }, + { Bucket, Key: '!exclamationPointObjTitle/' }, + { Bucket, Key: '!exclamationPointObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '!exclamationPointObjTitle/!exclamationPointObjTitle', Body: '{}' }, + { Bucket, Key: '-dashObjTitle/' }, + { Bucket, Key: '-dashObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '-dashObjTitle/-dashObjTitle', Body: '{}' }, + { Bucket, Key: '_underscoreObjTitle/' }, + { Bucket, Key: '_underscoreObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '_underscoreObjTitle/_underscoreObjTitle', Body: '{}' }, + { Bucket, Key: "'apostropheObjTitle/" }, + { Bucket, Key: "'apostropheObjTitle/objTitleA", Body: '{}' }, + { Bucket, Key: "'apostropheObjTitle/'apostropheObjTitle", Body: '{}' }, + { Bucket, Key: 'çcedilleObjTitle' }, + { Bucket, Key: 'çcedilleObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'çcedilleObjTitle/çcedilleObjTitle', Body: '{}' }, + { Bucket, Key: 'дcyrillicDObjTitle' }, + { Bucket, Key: 'дcyrillicDObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'дcyrillicDObjTitle/дcyrillicDObjTitle', Body: '{}' }, + { Bucket, Key: 'ñenyeObjTitle' }, + { Bucket, Key: 'ñenyeObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'ñenyeObjTitle/ñenyeObjTitle', Body: '{}' }, + { Bucket, Key: '山chineseMountainObjTitle' }, + { Bucket, Key: '山chineseMountainObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '山chineseMountainObjTitle/山chineseMountainObjTitle', Body: '{}' }, + { Bucket, Key: 'àaGraveLowerCaseObjTitle' }, + { Bucket, Key: 'àaGraveLowerCaseObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', Body: '{}' }, + { Bucket, Key: 'ÀaGraveUpperCaseObjTitle' }, + { Bucket, Key: 'ÀaGraveUpperCaseObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', Body: '{}' }, + { Bucket, Key: 'ßscharfesSObjTitle' }, + { Bucket, Key: 'ßscharfesSObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'ßscharfesSObjTitle/ßscharfesSObjTitle', Body: '{}' }, + { Bucket, Key: '日japaneseMountainObjTitle' }, + { Bucket, Key: '日japaneseMountainObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: '日japaneseMountainObjTitle/日japaneseMountainObjTitle', Body: '{}' }, + { Bucket, Key: 'بbaArabicObjTitle' }, + { Bucket, Key: 'بbaArabicObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'بbaArabicObjTitle/بbaArabicObjTitle', Body: '{}' }, + { Bucket, Key: 'अadevanagariHindiObjTitle' }, + { Bucket, Key: 'अadevanagariHindiObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', Body: '{}' }, + { Bucket, Key: 'éeacuteLowerCaseObjTitle' }, + { Bucket, Key: 'éeacuteLowerCaseObjTitle/objTitleA', Body: '{}' }, + { Bucket, Key: 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', Body: '{}' }, + ], listObjectParams: Bucket => ({ Bucket }), assertions: (data, Bucket) => { const keys = data.Contents.map(object => object.Key); assert.equal(data.Name, Bucket, 'Bucket name mismatch'); - assert.deepEqual(keys, [ - /* These object names are intentionally listed in a + assert.deepEqual( + keys, + [ + /* These object names are intentionally listed in a different order than they were created to additionally test that they are listed alphabetically. */ - '!exclamationPointObjTitle/', - '!exclamationPointObjTitle/!exclamationPointObjTitle', - '!exclamationPointObjTitle/objTitleA', - "'apostropheObjTitle/", - "'apostropheObjTitle/'apostropheObjTitle", - "'apostropheObjTitle/objTitleA", - '(openParenObjTitle/', - '(openParenObjTitle/(openParenObjTitle', - '(openParenObjTitle/objTitleA', - ')closeParenObjTitle/', - ')closeParenObjTitle/)closeParenObjTitle', - ')closeParenObjTitle/objTitleA', - '*asterixObjTitle/', - '*asterixObjTitle/*asterixObjTitle', - '*asterixObjTitle/objTitleA', - '-dashObjTitle/', - '-dashObjTitle/-dashObjTitle', - '-dashObjTitle/objTitleA', - '.dotObjTitle/', - '.dotObjTitle/.dotObjTitle', - '.dotObjTitle/objTitleA', - '_underscoreObjTitle/', - '_underscoreObjTitle/_underscoreObjTitle', - '_underscoreObjTitle/objTitleA', - 'foo&<>\'"', - 'ÀaGraveUpperCaseObjTitle', - 'ÀaGraveUpperCaseObjTitle/objTitleA', - 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', - 'ßscharfesSObjTitle', - 'ßscharfesSObjTitle/objTitleA', - 'ßscharfesSObjTitle/ßscharfesSObjTitle', - 'àaGraveLowerCaseObjTitle', - 'àaGraveLowerCaseObjTitle/objTitleA', - 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', - 'çcedilleObjTitle', - 'çcedilleObjTitle/objTitleA', - 'çcedilleObjTitle/çcedilleObjTitle', - 'éeacuteLowerCaseObjTitle', - 'éeacuteLowerCaseObjTitle/objTitleA', - 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', - 'ñenyeObjTitle', - 'ñenyeObjTitle/objTitleA', - 'ñenyeObjTitle/ñenyeObjTitle', - 'дcyrillicDObjTitle', - 'дcyrillicDObjTitle/objTitleA', - 'дcyrillicDObjTitle/дcyrillicDObjTitle', - 'بbaArabicObjTitle', - 'بbaArabicObjTitle/objTitleA', - 'بbaArabicObjTitle/بbaArabicObjTitle', - 'अadevanagariHindiObjTitle', - 'अadevanagariHindiObjTitle/objTitleA', - 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', - '山chineseMountainObjTitle', - '山chineseMountainObjTitle/objTitleA', - '山chineseMountainObjTitle/山chineseMountainObjTitle', - '日japaneseMountainObjTitle', - '日japaneseMountainObjTitle/objTitleA', - '日japaneseMountainObjTitle/日japaneseMountainObjTitle', - ], 'Bucket content mismatch'); + '!exclamationPointObjTitle/', + '!exclamationPointObjTitle/!exclamationPointObjTitle', + '!exclamationPointObjTitle/objTitleA', + "'apostropheObjTitle/", + "'apostropheObjTitle/'apostropheObjTitle", + "'apostropheObjTitle/objTitleA", + '(openParenObjTitle/', + '(openParenObjTitle/(openParenObjTitle', + '(openParenObjTitle/objTitleA', + ')closeParenObjTitle/', + ')closeParenObjTitle/)closeParenObjTitle', + ')closeParenObjTitle/objTitleA', + '*asterixObjTitle/', + '*asterixObjTitle/*asterixObjTitle', + '*asterixObjTitle/objTitleA', + '-dashObjTitle/', + '-dashObjTitle/-dashObjTitle', + '-dashObjTitle/objTitleA', + '.dotObjTitle/', + '.dotObjTitle/.dotObjTitle', + '.dotObjTitle/objTitleA', + '_underscoreObjTitle/', + '_underscoreObjTitle/_underscoreObjTitle', + '_underscoreObjTitle/objTitleA', + 'foo&<>\'"', + 'ÀaGraveUpperCaseObjTitle', + 'ÀaGraveUpperCaseObjTitle/objTitleA', + 'ÀaGraveUpperCaseObjTitle/ÀaGraveUpperCaseObjTitle', + 'ßscharfesSObjTitle', + 'ßscharfesSObjTitle/objTitleA', + 'ßscharfesSObjTitle/ßscharfesSObjTitle', + 'àaGraveLowerCaseObjTitle', + 'àaGraveLowerCaseObjTitle/objTitleA', + 'àaGraveLowerCaseObjTitle/àaGraveLowerCaseObjTitle', + 'çcedilleObjTitle', + 'çcedilleObjTitle/objTitleA', + 'çcedilleObjTitle/çcedilleObjTitle', + 'éeacuteLowerCaseObjTitle', + 'éeacuteLowerCaseObjTitle/objTitleA', + 'éeacuteLowerCaseObjTitle/éeacuteLowerCaseObjTitle', + 'ñenyeObjTitle', + 'ñenyeObjTitle/objTitleA', + 'ñenyeObjTitle/ñenyeObjTitle', + 'дcyrillicDObjTitle', + 'дcyrillicDObjTitle/objTitleA', + 'дcyrillicDObjTitle/дcyrillicDObjTitle', + 'بbaArabicObjTitle', + 'بbaArabicObjTitle/objTitleA', + 'بbaArabicObjTitle/بbaArabicObjTitle', + 'अadevanagariHindiObjTitle', + 'अadevanagariHindiObjTitle/objTitleA', + 'अadevanagariHindiObjTitle/अadevanagariHindiObjTitle', + '山chineseMountainObjTitle', + '山chineseMountainObjTitle/objTitleA', + '山chineseMountainObjTitle/山chineseMountainObjTitle', + '日japaneseMountainObjTitle', + '日japaneseMountainObjTitle/objTitleA', + '日japaneseMountainObjTitle/日japaneseMountainObjTitle', + ], + 'Bucket content mismatch', + ); }, }, { name: 'list objects with special chars in CommonPrefixes', - objectPutParams: Bucket => - [ - { Bucket, Key: '&#' }, - { Bucket, Key: '"quot#' }, { Bucket, Key: '\'apos#' }, - { Bucket, Key: ' [ + { Bucket, Key: '&#' }, + { Bucket, Key: '"quot#' }, + { Bucket, Key: "'apos#" }, + { Bucket, Key: ' ({ Bucket, Delimiter: '#' }), assertions: data => { assert.deepStrictEqual(data.CommonPrefixes, [ - { Prefix: '"quot#' }, { Prefix: '&#' }, - { Prefix: '\'apos#' }, { Prefix: ' { before(done => { authenticatedBucketUtil = new BucketUtility('default', {}); unauthenticatedBucketUtil = new BucketUtility('default', {}, true); - authenticatedBucketUtil.createRandom(1) - .then(created => { - bucketName = created; - done(); - }) - .catch(done); + authenticatedBucketUtil + .createRandom(1) + .then(created => { + bucketName = created; + done(); + }) + .catch(done); }); after(done => { - authenticatedBucketUtil.deleteOne(bucketName) - .then(() => done()) - .catch(done); + authenticatedBucketUtil + .deleteOne(bucketName) + .then(() => done()) + .catch(done); }); it('should return 403 and AccessDenied on a private bucket', done => { const params = { Bucket: bucketName }; - unauthenticatedBucketUtil.s3.send(new ListObjectsCommand(params)) + unauthenticatedBucketUtil.s3 + .send(new ListObjectsCommand(params)) .then(() => { assert.fail('Expected request to fail with AccessDenied'); }) @@ -351,12 +318,13 @@ describe('GET Bucket - AWS.S3.listObjects', () => { before(done => { bucketUtil = new BucketUtility('default', sigCfg); - bucketUtil.createRandom(1) - .then(created => { - bucketName = created; - done(); - }) - .catch(done); + bucketUtil + .createRandom(1) + .then(created => { + bucketName = created; + done(); + }) + .catch(done); }); after(() => bucketUtil.deleteOne(bucketName)); @@ -373,7 +341,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsCommand(test.listObjectParams(Bucket))); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -395,7 +363,9 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsV2Command(test.listObjectParams(Bucket))); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { @@ -406,19 +376,19 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as Prefix`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; for (const param of objects) { - await s3.send(new PutObjectCommand(param)); - } + await s3.send(new PutObjectCommand(param)); + } const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, Prefix: k })); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -429,7 +399,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as Marker`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; @@ -441,7 +411,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, Marker: k })); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -452,21 +422,25 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as NextMarker`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }]; + const objects = [ + { Bucket, Key: k }, + { Bucket, Key: 'zzz' }, + ]; for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, MaxKeys: 1, - Delimiter: 'foo' })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ Bucket, MaxKeys: 1, Delimiter: 'foo' }), + ); const validationSchema = { ...bucketSchema, - required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchema.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)), }; const isValidResponse = tv4.validate(data, validationSchema); if (!isValidResponse) { @@ -477,7 +451,7 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { it(`should list objects with key ${k} as StartAfter`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; @@ -489,7 +463,9 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const { $metadata, ...data } = await s3.send(new ListObjectsV2Command({ Bucket, StartAfter: k })); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { @@ -500,9 +476,8 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as ContinuationToken`, - async () => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { + it(`should list objects with key ${k} as ContinuationToken`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; const objects = [{ Bucket, Key: k }]; @@ -510,46 +485,53 @@ describe('GET Bucket - AWS.S3.listObjects', () => { for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const { $metadata, ...data } = await s3.send(new ListObjectsV2Command({ - Bucket, - ContinuationToken: generateToken(k), - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsV2Command({ + Bucket, + ContinuationToken: generateToken(k), + }), + ); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { throw new Error(tv4.error); } - assert.deepStrictEqual( - decryptToken(data.ContinuationToken), k); + assert.deepStrictEqual(decryptToken(data.ContinuationToken), k); assert.strictEqual($metadata.httpStatusCode, 200); }); }); - ['&', '"quot', '\'apos', 'gt'].forEach(k => { - it(`should list objects with key ${k} as NextContinuationToken`, - async () => { + ['&', '"quot', "'apos", 'gt'].forEach(k => { + it(`should list objects with key ${k} as NextContinuationToken`, async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - const objects = [{ Bucket, Key: k }, { Bucket, Key: 'zzz' }]; + const objects = [ + { Bucket, Key: k }, + { Bucket, Key: 'zzz' }, + ]; for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const { $metadata, ...data } = await s3.send(new ListObjectsV2Command({ Bucket, MaxKeys: 1, - Delimiter: 'foo' })); + const { $metadata, ...data } = await s3.send( + new ListObjectsV2Command({ Bucket, MaxKeys: 1, Delimiter: 'foo' }), + ); const validationSchema2 = { ...bucketSchemaV2, - required: bucketSchemaV2.required.filter(field => Object.prototype.hasOwnProperty.call(data, field)) + required: bucketSchemaV2.required.filter(field => + Object.prototype.hasOwnProperty.call(data, field), + ), }; const isValidResponse = tv4.validate(data, validationSchema2); if (!isValidResponse) { throw new Error(tv4.error); } - assert.strictEqual( - decryptToken(data.NextContinuationToken), k); + assert.strictEqual(decryptToken(data.NextContinuationToken), k); assert.strictEqual($metadata.httpStatusCode, 200); }); }); @@ -565,31 +547,37 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const iamClient = new IAMClient(iamConfig); before(async () => { - const policyRes = await iamClient.send(new CreatePolicyCommand({ - PolicyName: 'bp-bypass-policy', - PolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'AllowS3ListBucket', - Effect: 'Allow', - Action: [ - 's3:ListBucket', + const policyRes = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: 'bp-bypass-policy', + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'AllowS3ListBucket', + Effect: 'Allow', + Action: ['s3:ListBucket'], + Resource: ['*'], + }, ], - Resource: ['*'], - }], + }), }), - })); + ); policyWithListBucketOnly = policyRes.Policy; const userRes = await iamClient.send(new CreateUserCommand({ UserName: 'user-without-permission' })); userWithListBucketOnly = userRes.User; - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userWithListBucketOnly.UserName, - PolicyArn: policyWithListBucketOnly.Arn, - })); - - const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({ - UserName: userWithListBucketOnly.UserName, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userWithListBucketOnly.UserName, + PolicyArn: policyWithListBucketOnly.Arn, + }), + ); + + const accessKeyRes = await iamClient.send( + new CreateAccessKeyCommand({ + UserName: userWithListBucketOnly.UserName, + }), + ); const accessKey = accessKeyRes.AccessKey; const s3Config = getConfig('default', { credentials: { @@ -601,10 +589,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); after(async () => { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: userWithListBucketOnly.UserName, - PolicyArn: policyWithListBucketOnly.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: userWithListBucketOnly.UserName, + PolicyArn: policyWithListBucketOnly.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyWithListBucketOnly.Arn })); await iamClient.send(new DeleteUserCommand({ UserName: userWithListBucketOnly.UserName })); }); @@ -613,18 +603,22 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'super-power-object', - Metadata: { - department: 'sales', - hr: 'true', - }, - })); - const result = await s3.send(new ListObjectsV2ExtendedCommand({ - Bucket, - ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'super-power-object', + Metadata: { + department: 'sales', + hr: 'true', + }, + }), + ); + const result = await s3.send( + new ListObjectsV2ExtendedCommand({ + Bucket, + ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], + }), + ); assert.strictEqual(result.Contents.length, 1); assert.strictEqual(result.Contents[0].Key, 'super-power-object'); @@ -636,20 +630,24 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'super-power-object', - Metadata: { - department: 'sales', - hr: 'true', - }, - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'super-power-object', + Metadata: { + department: 'sales', + hr: 'true', + }, + }), + ); try { - await s3ClientWithListBucketOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket, - ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], - })); + await s3ClientWithListBucketOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket, + ObjectAttributes: ['x-amz-meta-*', 'RestoreStatus', 'x-amz-meta-department'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -664,18 +662,22 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'super-power-object', - Metadata: { - department: 'sales', - hr: 'true', - }, - })); - const result = await s3ClientWithListBucketOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket, - ObjectAttributes: ['RestoreStatus'], - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'super-power-object', + Metadata: { + department: 'sales', + hr: 'true', + }, + }), + ); + const result = await s3ClientWithListBucketOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket, + ObjectAttributes: ['RestoreStatus'], + }), + ); assert.strictEqual(result.Contents.length, 1); assert.strictEqual(result.Contents[0].Key, 'super-power-object'); @@ -698,22 +700,28 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const setupIamUser = async (userName, policyDoc) => { let policy; if (policyDoc) { - const res = await iamClient.send(new CreatePolicyCommand({ - PolicyName: `${userName}-policy`, - PolicyDocument: JSON.stringify(policyDoc), - })); + const res = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: `${userName}-policy`, + PolicyDocument: JSON.stringify(policyDoc), + }), + ); policy = res.Policy; } const userRes = await iamClient.send(new CreateUserCommand({ UserName: userName })); if (policy) { - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policy.Arn, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policy.Arn, + }), + ); } - const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({ - UserName: userName, - })); + const accessKeyRes = await iamClient.send( + new CreateAccessKeyCommand({ + UserName: userName, + }), + ); const ak = accessKeyRes.AccessKey; const s3Cfg = getConfig('default', { credentials: { @@ -726,10 +734,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { const teardownIamUser = async (user, policy) => { if (policy) { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: user.UserName, - PolicyArn: policy.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: user.UserName, + PolicyArn: policy.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policy.Arn })); } await iamClient.send(new DeleteUserCommand({ UserName: user.UserName })); @@ -742,18 +752,20 @@ describe('GET Bucket - AWS.S3.listObjects', () => { s3: s3ClientOptAttrsOnly, } = await setupIamUser('user-opt-attrs-only', { Version: '2012-10-17', - Statement: [{ - Sid: 'AllowOptAttrsOnly', - Effect: 'Allow', - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: ['*'], - }], + Statement: [ + { + Sid: 'AllowOptAttrsOnly', + Effect: 'Allow', + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: ['*'], + }, + ], })); - ({ - user: userNoPermissions, - s3: s3ClientNoPermissions, - } = await setupIamUser('user-no-permissions', null)); + ({ user: userNoPermissions, s3: s3ClientNoPermissions } = await setupIamUser( + 'user-no-permissions', + null, + )); ({ user: userListAllowAttrsDeny, @@ -786,10 +798,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { it('should reject when user has only the new permission and not s3:ListBucket', async () => { try { - await s3ClientOptAttrsOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientOptAttrsOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -802,10 +816,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { it('should reject when user has neither permission', async () => { try { - await s3ClientNoPermissions.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientNoPermissions.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -818,10 +834,12 @@ describe('GET Bucket - AWS.S3.listObjects', () => { it('should reject when explicit deny on the new permission overrides allow', async () => { try { - await s3ClientListAllowAttrsDeny.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientListAllowAttrsDeny.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { @@ -840,34 +858,40 @@ describe('GET Bucket - AWS.S3.listObjects', () => { before(async () => { const userName = 'user-with-both-perms'; - const policyRes = await iamClient.send(new CreatePolicyCommand({ - PolicyName: `${userName}-policy`, - PolicyDocument: JSON.stringify({ - Version: '2012-10-17', - Statement: [ - { - Effect: 'Allow', - Action: ['s3:ListBucket'], - Resource: ['*'], - }, - { - Effect: 'Allow', - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: ['*'], - }, - ], + const policyRes = await iamClient.send( + new CreatePolicyCommand({ + PolicyName: `${userName}-policy`, + PolicyDocument: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Action: ['s3:ListBucket'], + Resource: ['*'], + }, + { + Effect: 'Allow', + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: ['*'], + }, + ], + }), }), - })); + ); policyWithBothPerms = policyRes.Policy; const userRes = await iamClient.send(new CreateUserCommand({ UserName: userName })); userWithBothPerms = userRes.User; - await iamClient.send(new AttachUserPolicyCommand({ - UserName: userName, - PolicyArn: policyWithBothPerms.Arn, - })); - const accessKeyRes = await iamClient.send(new CreateAccessKeyCommand({ - UserName: userName, - })); + await iamClient.send( + new AttachUserPolicyCommand({ + UserName: userName, + PolicyArn: policyWithBothPerms.Arn, + }), + ); + const accessKeyRes = await iamClient.send( + new CreateAccessKeyCommand({ + UserName: userName, + }), + ); const ak = accessKeyRes.AccessKey; const s3Cfg = getConfig('default', { credentials: { @@ -879,48 +903,52 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); after(async () => { - await iamClient.send(new DetachUserPolicyCommand({ - UserName: userWithBothPerms.UserName, - PolicyArn: policyWithBothPerms.Arn, - })); + await iamClient.send( + new DetachUserPolicyCommand({ + UserName: userWithBothPerms.UserName, + PolicyArn: policyWithBothPerms.Arn, + }), + ); await iamClient.send(new DeletePolicyCommand({ PolicyArn: policyWithBothPerms.Arn })); await iamClient.send(new DeleteUserCommand({ UserName: userWithBothPerms.UserName })); }); afterEach(async () => { - await bucketUtil.s3 - .send(new DeleteBucketPolicyCommand({ Bucket: bucketName })) - .catch(() => {}); + await bucketUtil.s3.send(new DeleteBucketPolicyCommand({ Bucket: bucketName })).catch(() => {}); }); - // eslint-disable-next-line max-len it('should allow when the bucket policy supplies scality:ListBucketOptionalObjectAttributes that IAM lacks', async () => { - await bucketUtil.s3.send(new PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Effect: 'Allow', - Principal: { AWS: userWithListBucketOnly.Arn }, - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: [ - `arn:aws:s3:::${bucketName}`, - `arn:aws:s3:::${bucketName}/*`, + await bucketUtil.s3.send( + new PutBucketPolicyCommand({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: userWithListBucketOnly.Arn }, + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], + }, ], - }], + }), }), - })); + ); - await bucketUtil.s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: 'object-with-color', - Metadata: { color: 'red' }, - })); + await bucketUtil.s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: 'object-with-color', + Metadata: { color: 'red' }, + }), + ); - const result = await s3ClientWithListBucketOnly.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-color'], - })); + const result = await s3ClientWithListBucketOnly.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-color'], + }), + ); assert.ok(Array.isArray(result.Contents)); assert.strictEqual(result.Contents.length, 1); @@ -928,27 +956,30 @@ describe('GET Bucket - AWS.S3.listObjects', () => { }); it('should reject when the bucket policy denies the new action even if IAM allows it', async () => { - await bucketUtil.s3.send(new PutBucketPolicyCommand({ - Bucket: bucketName, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Effect: 'Deny', - Principal: { AWS: userWithBothPerms.Arn }, - Action: ['scality:ListBucketOptionalObjectAttributes'], - Resource: [ - `arn:aws:s3:::${bucketName}`, - `arn:aws:s3:::${bucketName}/*`, + await bucketUtil.s3.send( + new PutBucketPolicyCommand({ + Bucket: bucketName, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Effect: 'Deny', + Principal: { AWS: userWithBothPerms.Arn }, + Action: ['scality:ListBucketOptionalObjectAttributes'], + Resource: [`arn:aws:s3:::${bucketName}`, `arn:aws:s3:::${bucketName}/*`], + }, ], - }], + }), }), - })); + ); try { - await s3ClientWithBothPerms.send(new ListObjectsV2ExtendedCommand({ - Bucket: bucketName, - ObjectAttributes: ['x-amz-meta-foo'], - })); + await s3ClientWithBothPerms.send( + new ListObjectsV2ExtendedCommand({ + Bucket: bucketName, + ObjectAttributes: ['x-amz-meta-foo'], + }), + ); throw new Error('Request should have been rejected'); } catch (err) { if (err.message === 'Request should have been rejected') { diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js b/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js index 8327b1970d..c40ac495fe 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketEncryption.js @@ -1,8 +1,10 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - GetBucketEncryptionCommand } = require('@aws-sdk/client-s3'); + GetBucketEncryptionCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -38,9 +40,9 @@ describe('aws-sdk test get bucket encryption', () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); await new Promise((resolve, reject) => { - metadata.setup(err => err ? reject(err) : resolve()); + metadata.setup(err => (err ? reject(err) : resolve())); }); - }); + }); beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); @@ -75,8 +77,13 @@ describe('aws-sdk test get bucket encryption', () => { }); it('should include KMSMasterKeyID if user has configured a custom master key', async () => { - await setEncryptionInfo({ cryptoScheme: 1, algorithm: 'aws:kms', masterKeyId: '12345', - configuredMasterKeyId: '54321', mandatory: true }); + await setEncryptionInfo({ + cryptoScheme: 1, + algorithm: 'aws:kms', + masterKeyId: '12345', + configuredMasterKeyId: '54321', + mandatory: true, + }); const { $metadata, ...res } = await s3.send(new GetBucketEncryptionCommand({ Bucket: bucketName })); assert.deepStrictEqual(res, { ServerSideEncryptionConfiguration: { diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js b/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js index b0f7aaf270..1552b9850f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketLifecycle.js @@ -1,10 +1,12 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketLifecycleConfigurationCommand, - PutBucketLifecycleConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketLifecycleConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -15,11 +17,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -57,8 +64,7 @@ describe('aws-sdk test get bucket lifecycle', () => { } }); - it('should return NoSuchLifecycleConfiguration error if no lifecycle ' + - 'put to bucket', async () => { + it('should return NoSuchLifecycleConfiguration error if no lifecycle ' + 'put to bucket', async () => { try { await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); throw new Error('Expected NoSuchLifecycleConfiguration error'); @@ -68,17 +74,21 @@ describe('aws-sdk test get bucket lifecycle', () => { }); it('should get bucket lifecycle config with top-level prefix', async () => { - await s3.send(new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { - Rules: [{ - ID: 'test-id', - Status: 'Enabled', - Prefix: '', - Expiration: { Days: 1 }, - }], - }, - })); + await s3.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: bucket, + LifecycleConfiguration: { + Rules: [ + { + ID: 'test-id', + Status: 'Enabled', + Prefix: '', + Expiration: { Days: 1 }, + }, + ], + }, + }), + ); const res = await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); assert.strictEqual(res.Rules.length, 1); assert.deepStrictEqual(res.Rules[0], { @@ -90,17 +100,21 @@ describe('aws-sdk test get bucket lifecycle', () => { }); it('should get bucket lifecycle config with filter prefix', async () => { - await s3.send(new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { - Rules: [{ - ID: 'test-id', - Status: 'Enabled', - Filter: { Prefix: '' }, - Expiration: { Days: 1 }, - }], - }, - })); + await s3.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: bucket, + LifecycleConfiguration: { + Rules: [ + { + ID: 'test-id', + Status: 'Enabled', + Filter: { Prefix: '' }, + Expiration: { Days: 1 }, + }, + ], + }, + }), + ); const res = await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); assert.strictEqual(res.Rules.length, 1); assert.deepStrictEqual(res.Rules[0], { @@ -112,27 +126,31 @@ describe('aws-sdk test get bucket lifecycle', () => { }); it('should get bucket lifecycle config with filter prefix and tags', async () => { - await s3.send(new PutBucketLifecycleConfigurationCommand({ - Bucket: bucket, - LifecycleConfiguration: { - Rules: [{ - ID: 'test-id', - Status: 'Enabled', - Filter: { - And: { - Prefix: '', - Tags: [ - { - Key: 'key', - Value: 'value', + await s3.send( + new PutBucketLifecycleConfigurationCommand({ + Bucket: bucket, + LifecycleConfiguration: { + Rules: [ + { + ID: 'test-id', + Status: 'Enabled', + Filter: { + And: { + Prefix: '', + Tags: [ + { + Key: 'key', + Value: 'value', + }, + ], }, - ], + }, + Expiration: { Days: 1 }, }, - }, - Expiration: { Days: 1 }, - }], - }, - })); + ], + }, + }), + ); const res = await s3.send(new GetBucketLifecycleConfigurationCommand({ Bucket: bucket })); assert.strictEqual(res.Rules.length, 1); assert.deepStrictEqual(res.Rules[0], { diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js b/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js index dcdb101d75..1cd628c156 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketLogging.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - GetBucketLoggingCommand, - PutBucketLoggingCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, GetBucketLoggingCommand, PutBucketLoggingCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -30,7 +26,9 @@ function cleanUp(bucketUtil, cb) { throw err; } }), - ]).then(() => cb()).catch(err => cb(err)); + ]) + .then(() => cb()) + .catch(err => cb(err)); } describe('GET bucket logging', () => { @@ -38,10 +36,14 @@ describe('GET bucket logging', () => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - after(done => { cleanUp(bucketUtil, done); }); + after(done => { + cleanUp(bucketUtil, done); + }); describe('without existing bucket', () => { - afterEach(done => { cleanUp(bucketUtil, done); }); + afterEach(done => { + cleanUp(bucketUtil, done); + }); it('should return NoSuchBucket', done => { s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })) @@ -58,7 +60,9 @@ describe('GET bucket logging', () => { }); describe('on bucket without logging configuration', () => { - afterEach(done => { cleanUp(bucketUtil, done); }); + afterEach(done => { + cleanUp(bucketUtil, done); + }); beforeEach(done => { process.stdout.write('Creating bucket without logging\n'); @@ -86,16 +90,22 @@ describe('GET bucket logging', () => { }); describe('with existing logging configuration', () => { - afterEach(done => { cleanUp(bucketUtil, done); }); + afterEach(done => { + cleanUp(bucketUtil, done); + }); beforeEach(done => { process.stdout.write('Creating buckets and setting logging\n'); s3.send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => s3.send(new CreateBucketCommand({ Bucket: targetBucket }))) - .then(() => s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - }))) + .then(() => + s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ), + ) .then(() => done()) .catch(done); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js b/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js index 9ca98ef370..2f9a953d94 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketNotification.js @@ -1,21 +1,25 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketNotificationConfigurationCommand, - PutBucketNotificationConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketNotificationConfigurationCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'notificationtestbucket'; const notificationConfig = { - QueueConfigurations: [{ - Events: ['s3:ObjectCreated:*'], - QueueArn: 'arn:scality:bucketnotif:::target1', - Id: 'test-id', - }], + QueueConfigurations: [ + { + Events: ['s3:ObjectCreated:*'], + QueueArn: 'arn:scality:bucketnotif:::target1', + Id: 'test-id', + }, + ], }; // Check for the expected error response code and status code. @@ -61,14 +65,17 @@ describe('aws-sdk test get bucket notification', () => { } }); - it('should not return an error if no notification configuration ' + - 'put to bucket', () => s3.send(new GetBucketNotificationConfigurationCommand({ Bucket: bucket }))); + it('should not return an error if no notification configuration ' + 'put to bucket', () => + s3.send(new GetBucketNotificationConfigurationCommand({ Bucket: bucket })), + ); it('should get bucket notification config', async () => { - await s3.send(new PutBucketNotificationConfigurationCommand({ - Bucket: bucket, - NotificationConfiguration: notificationConfig, - })); + await s3.send( + new PutBucketNotificationConfigurationCommand({ + Bucket: bucket, + NotificationConfiguration: notificationConfig, + }), + ); const res = await s3.send(new GetBucketNotificationConfigurationCommand({ Bucket: bucket })); assert.deepStrictEqual(res.QueueConfigurations, notificationConfig.QueueConfigurations); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js b/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js index 11eaca40f6..9615fa8cc5 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketObjectLock.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetObjectLockConfigurationCommand, - PutObjectLockConfigurationCommand } = require('@aws-sdk/client-s3'); + PutObjectLockConfigurationCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -61,10 +63,14 @@ describe('aws-sdk test get bucket object lock', () => { }); describe('config rules', () => { - beforeEach(() => s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - }))); + beforeEach(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ), + ); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -78,10 +84,12 @@ describe('aws-sdk test get bucket object lock', () => { }); it('should get bucket object lock config', async () => { - await s3.send(new PutObjectLockConfigurationCommand({ - Bucket: bucket, - ObjectLockConfiguration: objectLockConfig, - })); + await s3.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucket, + ObjectLockConfiguration: objectLockConfig, + }), + ); const res = await s3.send(new GetObjectLockConfigurationCommand({ Bucket: bucket })); assert.deepStrictEqual(res.ObjectLockConfiguration, objectLockConfig); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js index 25b8d66eb2..1fd515b16f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketPolicy.js @@ -1,10 +1,12 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketPolicyCommand, - PutBucketPolicyCommand } = require('@aws-sdk/client-s3'); + PutBucketPolicyCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -12,13 +14,15 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'getbucketpolicy-testbucket'; const bucketPolicy = { Version: '2012-10-17', - Statement: [{ - Sid: 'testid', - Effect: 'Allow', - Principal: '*', - Action: 's3:putBucketPolicy', - Resource: `arn:aws:s3:::${bucket}`, - }], + Statement: [ + { + Sid: 'testid', + Effect: 'Allow', + Principal: '*', + Action: 's3:putBucketPolicy', + Resource: `arn:aws:s3:::${bucket}`, + }, + ], }; const expectedPolicy = { Sid: 'testid', @@ -32,11 +36,16 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, - 'incorrect error status code: should be 400 but got ' + - `'${err.$metadata.httpStatusCode}'`); + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, + 'incorrect error status code: should be 400 but got ' + `'${err.$metadata.httpStatusCode}'`, + ); } } @@ -78,10 +87,12 @@ describe('aws-sdk test get bucket policy', () => { }); it('should get bucket policy', async () => { - await s3.send(new PutBucketPolicyCommand({ - Bucket: bucket, - Policy: JSON.stringify(bucketPolicy), - })); + await s3.send( + new PutBucketPolicyCommand({ + Bucket: bucket, + Policy: JSON.stringify(bucketPolicy), + }), + ); const res = await s3.send(new GetBucketPolicyCommand({ Bucket: bucket })); const parsedRes = JSON.parse(res.Policy); assert.deepStrictEqual(parsedRes.Statement[0], expectedPolicy); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js b/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js index 35fd316623..815c606f0b 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketQuota.js @@ -1,6 +1,4 @@ -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); const sendRequest = require('../quota/tooling').sendRequest; @@ -49,7 +47,7 @@ describe('Test get bucket quota', () => { await sendRequest('GET', '127.0.0.1:8000', `/${bucket}/?quota=true`); assert.fail('Expected NoSuchQuota error'); } catch (err) { - assert.strictEqual(err.Error.Code[0], 'NoSuchQuota'); + assert.strictEqual(err.Error.Code[0], 'NoSuchQuota'); } }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js b/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js index 178539ed02..f2e840aaa0 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketRateLimit.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - S3Client, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const { sendRateLimitRequest, skipIfRateLimitDisabled } = require('../rateLimit/tooling'); @@ -35,12 +31,15 @@ skipIfRateLimitDisabled('Test get bucket rate limit', () => { it('should return the rate limit config', async () => { try { // First set the rate limit config - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); // Then get it - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, 100); } catch (err) { assert.ifError(err); diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js b/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js index 149d4c8511..f1e3b9412e 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketReplication.js @@ -1,10 +1,12 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketReplicationCommand, PutBucketReplicationCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); const getConfig = require('../support/config'); @@ -13,8 +15,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucket = 'source-bucket'; const replicationConfig = { - Role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + Role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', Rules: [ { Destination: { Bucket: 'arn:aws:s3:::destination-bucket' }, @@ -32,33 +33,40 @@ describe('aws-node-sdk test getBucketReplication', () => { beforeEach(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); - otherAccountS3 = new BucketUtility('lisa', {}).s3; + otherAccountS3 = new BucketUtility('lisa', {}).s3; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); }); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - it("should return 'ReplicationConfigurationNotFoundError' if bucket does " + - 'not have a replication configuration', async () => { - try { - await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); - throw new Error('Expected ReplicationConfigurationNotFoundError'); - } catch (err) { - assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.Code]); - } - }); + it( + "should return 'ReplicationConfigurationNotFoundError' if bucket does " + + 'not have a replication configuration', + async () => { + try { + await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); + throw new Error('Expected ReplicationConfigurationNotFoundError'); + } catch (err) { + assert(errorInstances.ReplicationConfigurationNotFoundError.is[err.Code]); + } + }, + ); it('should get the replication configuration that was put on a bucket', async () => { - await s3.send(new PutBucketReplicationCommand({ - Bucket: bucket, - ReplicationConfiguration: replicationConfig, - })); + await s3.send( + new PutBucketReplicationCommand({ + Bucket: bucket, + ReplicationConfiguration: replicationConfig, + }), + ); const data = await s3.send(new GetBucketReplicationCommand({ Bucket: bucket })); const expectedObj = { ReplicationConfiguration: replicationConfig, diff --git a/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js b/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js index 1b0605709d..2a2e1f24eb 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js +++ b/tests/functional/aws-node-sdk/test/bucket/getBucketTagging.js @@ -1,9 +1,11 @@ const assertError = require('../../../../utilities/bucketTagging-util'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketTaggingCommand, - PutBucketTaggingCommand } = require('@aws-sdk/client-s3'); + PutBucketTaggingCommand, +} = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -19,16 +21,18 @@ describe('aws-sdk test get bucket tagging', () => { }); beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); - + afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); it('should return accessDenied if expected bucket owner does not match', async () => { try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - ExpectedBucketOwner: '944690102203', - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + ExpectedBucketOwner: '944690102203', + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { assertError(err, 'AccessDenied'); @@ -37,11 +41,13 @@ describe('aws-sdk test get bucket tagging', () => { it('should not return accessDenied if expected bucket owner matches', async () => { try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -57,17 +63,21 @@ describe('aws-sdk test get bucket tagging', () => { }, ], }; - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: tagSet, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); - const result = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: tagSet, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); + const result = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); assert.deepStrictEqual(result.TagSet, tagSet.TagSet); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getCors.js b/tests/functional/aws-node-sdk/test/bucket/getCors.js index ba8a1636bc..2573cca92f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getCors.js +++ b/tests/functional/aws-node-sdk/test/bucket/getCors.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, GetBucketCorsCommand, - PutBucketCorsCommand } = require('@aws-sdk/client-s3'); + PutBucketCorsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); @@ -18,24 +20,27 @@ describe('GET bucket cors', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); describe('on bucket with existing cors configuration', () => { - const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000, - ExposeHeaders: ['x-amz-server-side-encryption'] }, - { AllowedMethods: ['GET'], - AllowedOrigins: ['*'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000 }, - ] }; - + const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: ['*'], + MaxAgeSeconds: 3000, + ExposeHeaders: ['x-amz-server-side-encryption'], + }, + { AllowedMethods: ['GET'], AllowedOrigins: ['*'], AllowedHeaders: ['*'], MaxAgeSeconds: 3000 }, + ], + }; + before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors, - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should return cors configuration successfully', async () => { @@ -46,45 +51,50 @@ describe('GET bucket cors', () => { describe('mixed case for AllowedHeader', () => { const testValue = 'tEsTvAlUe'; - const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: [testValue] }, - ] }; - + const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: [testValue], + }, + ], + }; + before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors, - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should be preserved when putting / getting cors resource', async () => { const data = await s3.send(new GetBucketCorsCommand({ Bucket: bucketName })); - assert.deepStrictEqual(data.CORSRules[0].AllowedHeaders, - sampleCors.CORSRules[0].AllowedHeaders); + assert.deepStrictEqual(data.CORSRules[0].AllowedHeaders, sampleCors.CORSRules[0].AllowedHeaders); }); }); describe('uppercase for AllowedMethod', () => { - const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'] }, - ] }; - + const sampleCors = { + CORSRules: [{ AllowedMethods: ['PUT', 'POST', 'DELETE'], AllowedOrigins: ['http://www.example.com'] }], + }; + before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors, - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should be preserved when retrieving cors resource', async () => { const data = await s3.send(new GetBucketCorsCommand({ Bucket: bucketName })); - assert.deepStrictEqual(data.CORSRules[0].AllowedMethods, - sampleCors.CORSRules[0].AllowedMethods); + assert.deepStrictEqual(data.CORSRules[0].AllowedMethods, sampleCors.CORSRules[0].AllowedMethods); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/getLocation.js b/tests/functional/aws-node-sdk/test/bucket/getLocation.js index fc847c14ac..47efd6a29e 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getLocation.js +++ b/tests/functional/aws-node-sdk/test/bucket/getLocation.js @@ -6,9 +6,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const getConfig = require('../support/config'); const { config } = require('../../../../../lib/Config'); -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const bucketName = 'testgetlocationbucket'; @@ -22,8 +20,7 @@ describeSkipAWS('GET bucket location ', () => { const otherAccountBucketUtility = new BucketUtility('lisa', {}); const otherAccountS3 = otherAccountBucketUtility.s3; const locationConstraints = config.locationConstraints; - Object.keys(locationConstraints).forEach( - location => { + Object.keys(locationConstraints).forEach(location => { if (location === 'us-east-1') { // if location is us-east-1 should return empty string // see next test. @@ -38,16 +35,19 @@ describeSkipAWS('GET bucket location ', () => { return; } describe(`with location: ${location}`, () => { - before(() => s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: location, - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: location, + }, + }), + ), + ); after(() => bucketUtil.deleteOne(bucketName)); - it(`should return location configuration: ${location} ` + - 'successfully', async () => { + it(`should return location configuration: ${location} ` + 'successfully', async () => { const data = await s3.send(new GetBucketLocationCommand({ Bucket: bucketName })); assert.deepStrictEqual(data.LocationConstraint, location); }); @@ -55,14 +55,18 @@ describeSkipAWS('GET bucket location ', () => { }); describe('with location us-east-1', () => { - before(() => s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ), + ); afterEach(() => bucketUtil.deleteOne(bucketName)); - + it('should return empty location', async () => { const data = await s3.send(new GetBucketLocationCommand({ Bucket: bucketName })); // SDK v3 returns undefined for us-east-1, normalize to empty string for comparison @@ -74,20 +78,19 @@ describeSkipAWS('GET bucket location ', () => { describe('without location configuration', () => { after(() => { process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucketName) - .catch(err => { - process.stdout.write(`Error in after: ${err}\n`); - throw err; - }); + return bucketUtil.deleteOne(bucketName).catch(err => { + process.stdout.write(`Error in after: ${err}\n`); + throw err; + }); }); it('should return request endpoint as location', async () => { process.stdout.write('Creating bucket'); await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - + // In SDK v3, we need to get the endpoint from the client config let host = '127.0.0.1'; - + if (clientConfig.endpoint) { try { const url = new URL(clientConfig.endpoint); @@ -97,20 +100,20 @@ describeSkipAWS('GET bucket location ', () => { host = clientConfig.endpoint; } } - + let endpoint = config.restEndpoints[host]; // s3 actually returns '' for us-east-1 if (endpoint === 'us-east-1') { endpoint = ''; } - + const data = await s3.send(new GetBucketLocationCommand({ Bucket: bucketName })); - + // S3C backend has 'dc-1' as default location constraint // Other backends use endpoint-based location const isS3C = process.env.S3BACKEND === 's3c'; const expectedLocation = isS3C ? 'dc-1' : endpoint; - + const actualLocation = data.LocationConstraint || ''; const normalizedExpected = expectedLocation || ''; assert.strictEqual(actualLocation, normalizedExpected); @@ -118,12 +121,16 @@ describeSkipAWS('GET bucket location ', () => { }); describe('with location configuration', () => { - before(() => s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ), + ); after(() => bucketUtil.deleteOne(bucketName)); it('should return AccessDenied if user is not bucket owner', async () => { diff --git a/tests/functional/aws-node-sdk/test/bucket/getWebsite.js b/tests/functional/aws-node-sdk/test/bucket/getWebsite.js index 766235d37c..458919cab3 100644 --- a/tests/functional/aws-node-sdk/test/bucket/getWebsite.js +++ b/tests/functional/aws-node-sdk/test/bucket/getWebsite.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, GetBucketWebsiteCommand, PutBucketWebsiteCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); @@ -32,15 +34,17 @@ describe('GET bucket website', () => { const s3Config = getConfig('default', sigCfg); const s3 = new S3Client(s3Config); - afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); + afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); describe('with existing bucket configuration', () => { before(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketWebsiteCommand({ - Bucket: bucketName, - WebsiteConfiguration: config, - })); + await s3.send( + new PutBucketWebsiteCommand({ + Bucket: bucketName, + WebsiteConfiguration: config, + }), + ); }); it('should return bucket website xml successfully', async () => { diff --git a/tests/functional/aws-node-sdk/test/bucket/head.js b/tests/functional/aws-node-sdk/test/bucket/head.js index af8d18f854..64c28fcbd0 100644 --- a/tests/functional/aws-node-sdk/test/bucket/head.js +++ b/tests/functional/aws-node-sdk/test/bucket/head.js @@ -4,7 +4,6 @@ const { S3Client, HeadBucketCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); - describe('HEAD bucket', () => { withV4(sigCfg => { let s3; @@ -14,16 +13,14 @@ describe('HEAD bucket', () => { s3 = new S3Client(config); }); - it('should return an error to a head request without a ' + - 'bucket name', - async () => { - try { - await s3.send(new HeadBucketCommand({ Bucket: '' })); - assert.fail('Expected failure but got success'); - } catch (err) { - assert.strictEqual(err.$metadata.httpStatusCode, 405); - assert.strictEqual(err.name, 'Unknown'); - } - }); + it('should return an error to a head request without a ' + 'bucket name', async () => { + try { + await s3.send(new HeadBucketCommand({ Bucket: '' })); + assert.fail('Expected failure but got success'); + } catch (err) { + assert.strictEqual(err.$metadata.httpStatusCode, 405); + assert.strictEqual(err.name, 'Unknown'); + } + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js b/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js index 3f86f88b53..975a5f69af 100644 --- a/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js +++ b/tests/functional/aws-node-sdk/test/bucket/listingCornerCases.js @@ -1,11 +1,13 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, DeleteBucketCommand, ListObjectsCommand, ListObjectsV2Command, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -60,7 +62,7 @@ const allKeys = objects.map(obj => obj.Key); describe('Listing corner cases tests', () => { let s3; - + before(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3 = new S3Client(config); @@ -83,17 +85,19 @@ describe('Listing corner cases tests', () => { Marker: '', MaxKeys: 1000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with valid marker', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Marker: 'notes/summer/1.txt', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Marker: 'notes/summer/1.txt', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Delimiter: '/', @@ -101,17 +105,19 @@ describe('Listing corner cases tests', () => { Marker: 'notes/summer/1.txt', MaxKeys: 1000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with unexpected marker', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Marker: 'zzzz', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Marker: 'zzzz', + }), + ); assert.deepStrictEqual(data, { IsTruncated: false, Marker: 'zzzz', @@ -124,12 +130,14 @@ describe('Listing corner cases tests', () => { }); it('should list with unexpected marker and prefix', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Marker: 'notes/summer0', - Prefix: 'notes/summer/', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Marker: 'notes/summer0', + Prefix: 'notes/summer/', + }), + ); assert.deepStrictEqual(data, { IsTruncated: false, Marker: 'notes/summer0', @@ -142,27 +150,31 @@ describe('Listing corner cases tests', () => { }); it('should list with MaxKeys', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - MaxKeys: 3, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + MaxKeys: 3, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Contents: objects.slice(0, 3).map(obj => obj.Key), IsTruncated: true, Marker: '', - MaxKeys: 3, + MaxKeys: 3, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with big MaxKeys', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - MaxKeys: 15000, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + MaxKeys: 15000, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Contents: allKeys, @@ -170,16 +182,18 @@ describe('Listing corner cases tests', () => { Marker: '', MaxKeys: 15000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with delimiter', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Contents: [objects[0].Key], @@ -189,16 +203,18 @@ describe('Listing corner cases tests', () => { Marker: '', MaxKeys: 1000, Name: Bucket, - Prefix: '' + Prefix: '', }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with long delimiter', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: 'notes/summer', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: 'notes/summer', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: '', @@ -214,11 +230,13 @@ describe('Listing corner cases tests', () => { }); it('should list with delimiter and prefix related to #147', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: '', @@ -228,22 +246,20 @@ describe('Listing corner cases tests', () => { Prefix: 'notes/', Delimiter: '/', MaxKeys: 1000, - CommonPrefixes: [ - 'notes/spring/', - 'notes/summer/', - 'notes/zaphod/', - ], + CommonPrefixes: ['notes/spring/', 'notes/summer/', 'notes/zaphod/'], }); assert.strictEqual($metadata.httpStatusCode, 200); }); it('should list with prefix and marker related to #147', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/year.txt', - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/year.txt', + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/year.txt', @@ -259,13 +275,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 1 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/', @@ -281,13 +299,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 2 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/spring/', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/spring/', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/spring/', @@ -303,13 +323,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 3 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/summer/', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/summer/', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/summer/', @@ -325,13 +347,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 4 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/year.txt', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/year.txt', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/year.txt', @@ -347,13 +371,15 @@ describe('Listing corner cases tests', () => { }); it('should list with all parameters 5 of 5', async () => { - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/yore.rs', - MaxKeys: 1, - })); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/yore.rs', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { Marker: 'notes/yore.rs', @@ -368,18 +394,22 @@ describe('Listing corner cases tests', () => { }); it('should end listing on last common prefix', async () => { - await s3.send(new PutObjectCommand({ - Bucket, - Key: 'notes/zaphod/TheFourth.txt', - Body: '', - })); - const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ - Bucket, - Delimiter: '/', - Prefix: 'notes/', - Marker: 'notes/yore.rs', - MaxKeys: 1, - })); + await s3.send( + new PutObjectCommand({ + Bucket, + Key: 'notes/zaphod/TheFourth.txt', + Body: '', + }), + ); + const { $metadata, ...data } = await s3.send( + new ListObjectsCommand({ + Bucket, + Delimiter: '/', + Prefix: 'notes/', + Marker: 'notes/yore.rs', + MaxKeys: 1, + }), + ); cutAttributes(data); assert.deepStrictEqual(data, { IsTruncated: false, @@ -396,49 +426,62 @@ describe('Listing corner cases tests', () => { it('should not list DeleteMarkers for version suspended buckets', async () => { const obj = { name: 'testDeleteMarker.txt', value: 'foo' }; const bucketName = `bucket-test-delete-markers-not-listed${Date.now()}`; - + try { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Suspended', - }, - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: obj.name, - Body: obj.value, - })); - + + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Suspended', + }, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: obj.name, + Body: obj.value, + }), + ); + const listRes1 = await s3.send(new ListObjectsV2Command({ Bucket: bucketName })); - assert.strictEqual(listRes1.Contents.some(c => c.Key === obj.name), true); - - const deleteRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.name, - })); + assert.strictEqual( + listRes1.Contents.some(c => c.Key === obj.name), + true, + ); + + const deleteRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.name, + }), + ); assert.strictEqual(deleteRes.DeleteMarker, true); - + const listRes2 = await s3.send(new ListObjectsV2Command({ Bucket: bucketName })); assert.strictEqual(listRes2.Contents, undefined); - - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.name, - VersionId: 'null' - })); - + + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.name, + VersionId: 'null', + }), + ); + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); } catch (err) { try { - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: obj.name, - VersionId: 'null' - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: obj.name, + VersionId: 'null', + }), + ); await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); } catch { // Ignore cleanup errors diff --git a/tests/functional/aws-node-sdk/test/bucket/put.js b/tests/functional/aws-node-sdk/test/bucket/put.js index 5773adbca1..3ffcc811d7 100644 --- a/tests/functional/aws-node-sdk/test/bucket/put.js +++ b/tests/functional/aws-node-sdk/test/bucket/put.js @@ -12,9 +12,7 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const withV4 = require('../support/withV4'); const configOfficial = require('../../../../../lib/Config').config; -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const bucketName = 'bucketlocation'; @@ -25,7 +23,6 @@ const locationConstraints = configOfficial.locationConstraints; describe('PUT Bucket - AWS.S3.createBucket', () => { describe('When user is unauthorized', () => { - it('should return 403 and AccessDenied', async () => { const params = { Bucket: 'mybucket' }; try { @@ -45,44 +42,51 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { before(() => { bucketUtil = new BucketUtility('default', sigCfg); }); - + describe('create bucket twice', () => { let testBucketName; - + beforeEach(() => { // Use unique bucket name for each test to avoid conflicts testBucketName = `${bucketName}-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`; - return bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: testBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - })); + return bucketUtil.s3.send( + new CreateBucketCommand({ + Bucket: testBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ); }); - + afterEach(() => bucketUtil.s3.send(new DeleteBucketCommand({ Bucket: testBucketName }))); - - itSkipIfE2E('should return a 200 if no locationConstraints provided.', - () => bucketUtil.s3.send(new CreateBucketCommand({ Bucket: testBucketName }))); + + itSkipIfE2E('should return a 200 if no locationConstraints provided.', () => + bucketUtil.s3.send(new CreateBucketCommand({ Bucket: testBucketName })), + ); it('should return a 200 if us-east behavior', async () => { - const res = await bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: testBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-1', - }, - })); + const res = await bucketUtil.s3.send( + new CreateBucketCommand({ + Bucket: testBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-1', + }, + }), + ); assert.strictEqual(res.$metadata.httpStatusCode, 200); }); - + it('should return a 409 if us-west behavior', async () => { try { - await bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: testBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'scality-us-west-1', - }, - })); + await bucketUtil.s3.send( + new CreateBucketCommand({ + Bucket: testBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'scality-us-west-1', + }, + }), + ); assert.fail('Expected failure but got success'); } catch (error) { assert.strictEqual(error.name, 'BucketAlreadyOwnedByYou'); @@ -101,15 +105,13 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { bucketUtil .createOne(bucketName) .then(() => { - const e = new Error('Expect failure in creation, ' + - 'but it succeeded'); + const e = new Error('Expect failure in creation, ' + 'but it succeeded'); return done(e); }) .catch(error => { assert.strictEqual(error.Code, expectedCode); - assert.strictEqual(error.$metadata.httpStatusCode, - expectedStatus); + assert.strictEqual(error.$metadata.httpStatusCode, expectedStatus); done(); }); }; @@ -133,45 +135,35 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { testFn(shortName, done); }); - it('should return 403 if name is reserved (e.g., METADATA)', - done => { - const reservedName = 'METADATA'; - testFn(reservedName, done, 403, 'AccessDenied'); - }); + it('should return 403 if name is reserved (e.g., METADATA)', done => { + const reservedName = 'METADATA'; + testFn(reservedName, done, 403, 'AccessDenied'); + }); - itSkipIfAWS('should return 400 if name is longer than 63 chars', - done => { - const longName = 'x'.repeat(64); - testFn(longName, done); - } - ); + itSkipIfAWS('should return 400 if name is longer than 63 chars', done => { + const longName = 'x'.repeat(64); + testFn(longName, done); + }); - itSkipIfAWS('should return 400 if name is formatted as IP address', - done => { - const ipAddress = '192.168.5.4'; - testFn(ipAddress, done); - } - ); + itSkipIfAWS('should return 400 if name is formatted as IP address', done => { + const ipAddress = '192.168.5.4'; + testFn(ipAddress, done); + }); - itSkipIfAWS('should return 400 if name starts with period', - done => { - const invalidName = '.myawsbucket'; - testFn(invalidName, done); - } - ); + itSkipIfAWS('should return 400 if name starts with period', done => { + const invalidName = '.myawsbucket'; + testFn(invalidName, done); + }); it('should return 400 if name ends with period', done => { const invalidName = 'myawsbucket.'; testFn(invalidName, done); }); - itSkipIfAWS( - 'should return 400 if name has two period between labels', - done => { - const invalidName = 'my..examplebucket'; - testFn(invalidName, done); - } - ); + itSkipIfAWS('should return 400 if name has two period between labels', done => { + const invalidName = 'my..examplebucket'; + testFn(invalidName, done); + }); it('should return 400 if name has special chars', done => { const invalidName = 'my.#s3bucket'; @@ -181,71 +173,89 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { describe('bucket creation success', () => { function _test(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ Bucket: name })) + bucketUtil.s3 + .send(new CreateBucketCommand({ Bucket: name })) .then(res => { assert(res.Location, 'No Location in response'); - assert.deepStrictEqual(res.Location, `/${name}`, - 'Wrong Location header'); + assert.deepStrictEqual(res.Location, `/${name}`, 'Wrong Location header'); return bucketUtil.deleteOne(name); }) .then(() => done()) .catch(done); } - it('should create bucket if name is valid', done => - _test('scality-very-valid-bucket-name', done)); + it('should create bucket if name is valid', done => _test('scality-very-valid-bucket-name', done)); - it('should create bucket if name is some prefix and an IP address', - done => _test('prefix-192.168.5.4', done)); + it('should create bucket if name is some prefix and an IP address', done => + _test('prefix-192.168.5.4', done)); - it('should create bucket if name is an IP address with some suffix', - done => _test('192.168.5.4-suffix', done)); + it('should create bucket if name is an IP address with some suffix', done => + _test('192.168.5.4-suffix', done)); }); describe('bucket creation success with object lock', () => { function _testObjectLockEnabled(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: name, - ObjectLockEnabledForBucket: true, - })).then(res => { - assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); - return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); - }).then(res => { - assert.deepStrictEqual(res.ObjectLockConfiguration, - { ObjectLockEnabled: 'Enabled' }); - return bucketUtil.deleteOne(name); - }).then(() => done()).catch(done); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: name, + ObjectLockEnabledForBucket: true, + }), + ) + .then(res => { + assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); + return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); + }) + .then(res => { + assert.deepStrictEqual(res.ObjectLockConfiguration, { ObjectLockEnabled: 'Enabled' }); + return bucketUtil.deleteOne(name); + }) + .then(() => done()) + .catch(done); } function _testObjectLockDisabled(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: name, - ObjectLockEnabledForBucket: false, - })).then(res => { - assert(res.Location, 'No Location in response'); - assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); - return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); - }).catch(err => { - assert.strictEqual(err.name, 'ObjectLockConfigurationNotFoundError'); - return bucketUtil.deleteOne(name); - }).then(() => done()).catch(done); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: name, + ObjectLockEnabledForBucket: false, + }), + ) + .then(res => { + assert(res.Location, 'No Location in response'); + assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); + return bucketUtil.s3.send(new GetObjectLockConfigurationCommand({ Bucket: name })); + }) + .catch(err => { + assert.strictEqual(err.name, 'ObjectLockConfigurationNotFoundError'); + return bucketUtil.deleteOne(name); + }) + .then(() => done()) + .catch(done); } function _testVersioning(name, done) { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: name, - ObjectLockEnabledForBucket: true, - })).then(res => { - assert(res.Location, 'No Location in response'); - assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); - return bucketUtil.s3.send(new GetBucketVersioningCommand({ Bucket: name })); - }).then(res => { - assert.strictEqual(res.Status, 'Enabled'); - assert.strictEqual(res.MFADelete, 'Disabled'); - return bucketUtil.deleteOne(name); - }).then(() => done()).catch(done); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: name, + ObjectLockEnabledForBucket: true, + }), + ) + .then(res => { + assert(res.Location, 'No Location in response'); + assert.strictEqual(res.Location, `/${name}`, 'Wrong Location header'); + return bucketUtil.s3.send(new GetBucketVersioningCommand({ Bucket: name })); + }) + .then(res => { + assert.strictEqual(res.Status, 'Enabled'); + assert.strictEqual(res.MFADelete, 'Disabled'); + return bucketUtil.deleteOne(name); + }) + .then(() => done()) + .catch(done); } - it('should create bucket without error', done => - _testObjectLockEnabled('bucket-with-object-lock', done)); + it('should create bucket without error', done => _testObjectLockEnabled('bucket-with-object-lock', done)); it('should create bucket with versioning enabled by default', done => _testVersioning('bucket-with-object-lock', done)); @@ -254,98 +264,121 @@ describe('PUT Bucket - AWS.S3.createBucket', () => { _testObjectLockDisabled('bucket-without-object-lock', done)); }); - Object.keys(locationConstraints).forEach( - location => { - describeSkipAWS(`bucket creation with location: ${location}`, - () => { + Object.keys(locationConstraints).forEach(location => { + describeSkipAWS(`bucket creation with location: ${location}`, () => { after(done => { bucketUtil.deleteOne(bucketName).finally(done); }); it(`should create bucket with location: ${location}`, done => { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: location, - }, - })).then(() => { - done(); - }).catch(err => { - if (location === LOCATION_NAME_DMF) { - assert.strictEqual( - err.name, - 'InvalidLocationConstraint' - ); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } - done(); - }); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: location, + }, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + if (location === LOCATION_NAME_DMF) { + assert.strictEqual(err.name, 'InvalidLocationConstraint'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } + done(); + }); }); }); }); describe('bucket creation with invalid location', () => { it('should return errors InvalidLocationConstraint', done => { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'coco', - }, - })).catch(err => { - assert.strictEqual( - err.name, - 'InvalidLocationConstraint' - ); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'coco', + }, + }), + ) + .catch(err => { + assert.strictEqual(err.name, 'InvalidLocationConstraint'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); }); it('should return error InvalidLocationConstraint for location constraint dmf', done => { - bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: LOCATION_NAME_DMF, - }, - })).catch(err => { - assert.strictEqual( - err.name, - 'InvalidLocationConstraint' - ); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: LOCATION_NAME_DMF, + }, + }), + ) + .catch(err => { + assert.strictEqual(err.name, 'InvalidLocationConstraint'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); }); }); describe('bucket creation with ingestion location', () => { - after(() => bucketUtil.s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); - + after(() => bucketUtil.s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); + it('should create bucket with location and ingestion', done => { - async.waterfall([ - next => bucketUtil.s3.send(new CreateBucketCommand({ - Bucket: bucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-2:ingest', - }, - })).then(res => { - assert.strictEqual(res.Location, `/${bucketName}`); - next(); - }).catch(next), - - next => bucketUtil.s3.send(new GetBucketLocationCommand({ - Bucket: bucketName, - })).then(res => { - assert.strictEqual(res.LocationConstraint, 'us-east-2'); - next(); - }).catch(next), - - next => bucketUtil.s3.send(new GetBucketVersioningCommand({ - Bucket: bucketName, - })).then(res => { - assert.strictEqual(res.Status, 'Enabled'); - next(); - }).catch(next), - ], done); + async.waterfall( + [ + next => + bucketUtil.s3 + .send( + new CreateBucketCommand({ + Bucket: bucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + }), + ) + .then(res => { + assert.strictEqual(res.Location, `/${bucketName}`); + next(); + }) + .catch(next), + + next => + bucketUtil.s3 + .send( + new GetBucketLocationCommand({ + Bucket: bucketName, + }), + ) + .then(res => { + assert.strictEqual(res.LocationConstraint, 'us-east-2'); + next(); + }) + .catch(next), + + next => + bucketUtil.s3 + .send( + new GetBucketVersioningCommand({ + Bucket: bucketName, + }), + ) + .then(res => { + assert.strictEqual(res.Status, 'Enabled'); + next(); + }) + .catch(next), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/putAcl.js b/tests/functional/aws-node-sdk/test/bucket/putAcl.js index dc482d7de0..98c9d1535f 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putAcl.js +++ b/tests/functional/aws-node-sdk/test/bucket/putAcl.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketAclCommand, - GetBucketAclCommand } = require('@aws-sdk/client-s3'); + GetBucketAclCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -53,8 +55,7 @@ describe('aws-node-sdk test bucket put acl', () => { throw new Error('accepted xml body larger than 512 KB'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'InvalidRequest'); + assert.strictEqual(error.name, 'InvalidRequest'); } }); }); @@ -68,26 +69,30 @@ describe('PUT Bucket ACL', () => { afterEach(() => bucketUtil.deleteOne(bucketName)); - it('should set multiple ACL permissions with same grantee specified' + - 'using email', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - GrantRead: 'emailAddress=sampleaccount1@sampling.com', - GrantWrite: 'emailAddress=sampleaccount1@sampling.com', - })); - const res = await s3.send(new GetBucketAclCommand({ - Bucket: bucketName, - })); + it('should set multiple ACL permissions with same grantee specified' + 'using email', async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + GrantRead: 'emailAddress=sampleaccount1@sampling.com', + GrantWrite: 'emailAddress=sampleaccount1@sampling.com', + }), + ); + const res = await s3.send( + new GetBucketAclCommand({ + Bucket: bucketName, + }), + ); assert.strictEqual(res.Grants.length, 2); }); - it('should return InvalidArgument if invalid grantee ' + - 'user ID provided in ACL header request', async () => { + it('should return InvalidArgument if invalid grantee ' + 'user ID provided in ACL header request', async () => { try { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - GrantRead: 'id=invalidUserID' - })); + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + GrantRead: 'id=invalidUserID', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.$metadata.httpStatusCode, 400); @@ -95,27 +100,28 @@ describe('PUT Bucket ACL', () => { } }); - it('should return InvalidArgument if invalid grantee ' + - 'user ID provided in ACL request body', async () => { + it('should return InvalidArgument if invalid grantee ' + 'user ID provided in ACL request body', async () => { try { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - AccessControlPolicy: { - Grants: [ - { - Grantee: { - Type: 'CanonicalUser', - ID: 'invalidUserID', + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + AccessControlPolicy: { + Grants: [ + { + Grantee: { + Type: 'CanonicalUser', + ID: 'invalidUserID', + }, + Permission: 'WRITE_ACP', }, - Permission: 'WRITE_ACP', - }], - Owner: { - DisplayName: 'Bart', - ID: '79a59df900b949e55d96a1e698fbace' + - 'dfd6e09d98eacf8f8d5218e7cd47ef2be', + ], + Owner: { + DisplayName: 'Bart', + ID: '79a59df900b949e55d96a1e698fbace' + 'dfd6e09d98eacf8f8d5218e7cd47ef2be', + }, }, - }, - })); + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.$metadata.httpStatusCode, 400); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js b/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js index 5bbc81af27..f6f3a5e973 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketLogging.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutBucketLoggingCommand, - GetBucketLoggingCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutBucketLoggingCommand, GetBucketLoggingCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -55,7 +51,9 @@ function cleanUp(bucketUtil, cb) { throw err; } }), - ]).then(() => cb()).catch(err => cb(err)); + ]) + .then(() => cb()) + .catch(err => cb(err)); } describe('PUT bucket logging', () => { @@ -67,10 +65,12 @@ describe('PUT bucket logging', () => { async function _testPutBucketLoggingError(account, config, statusCode, errMsg, cb) { try { - await account.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: config, - })); + await account.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: config, + }), + ); return cb(new Error('Expected error but found none')); } catch (err) { assert(err, 'Expected err but found none'); @@ -100,10 +100,12 @@ describe('PUT bucket logging', () => { afterEach(async () => { process.stdout.write('Deleting buckets\n'); try { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: {}, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: {}, + }), + ); } catch (err) { if (err.name !== 'NoSuchBucket' && err.code !== 'NoSuchBucket') { throw err; @@ -131,17 +133,17 @@ describe('PUT bucket logging', () => { }); it('should put bucket logging configuration successfully', async () => { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ); const data = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(data.LoggingEnabled); - assert.strictEqual(data.LoggingEnabled.TargetBucket, - targetBucket); - assert.strictEqual(data.LoggingEnabled.TargetPrefix, - 'logs/'); + assert.strictEqual(data.LoggingEnabled.TargetBucket, targetBucket); + assert.strictEqual(data.LoggingEnabled.TargetPrefix, 'logs/'); }); itSkipIfAWS('should return NotImplemented if TargetGrants is present', done => { @@ -149,18 +151,22 @@ describe('PUT bucket logging', () => { }); it('should disable logging with empty BucketLoggingStatus', async () => { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ); const enabled = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(enabled.LoggingEnabled); - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: {}, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: {}, + }), + ); const disabled = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(disabled); @@ -171,22 +177,23 @@ describe('PUT bucket logging', () => { _testPutBucketLoggingError(otherAccountS3, validLoggingConfig, 405, 'MethodNotAllowed', done); }); - it('should return InvalidTargetBucketForLogging if target bucket does not exist', - done => { - const invalidConfig = { - LoggingEnabled: { - TargetBucket: 'nonexistentbucket', - TargetPrefix: 'logs/', - }, - }; - _testPutBucketLoggingError(s3, invalidConfig, 400, 'InvalidTargetBucketForLogging', done); - }); + it('should return InvalidTargetBucketForLogging if target bucket does not exist', done => { + const invalidConfig = { + LoggingEnabled: { + TargetBucket: 'nonexistentbucket', + TargetPrefix: 'logs/', + }, + }; + _testPutBucketLoggingError(s3, invalidConfig, 400, 'InvalidTargetBucketForLogging', done); + }); it('should allow logging when target bucket is owned by same account', async () => { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: validLoggingConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: validLoggingConfig, + }), + ); const data = await s3.send(new GetBucketLoggingCommand({ Bucket: bucketName })); assert(data.LoggingEnabled); @@ -229,8 +236,7 @@ describe('PUT bucket logging', () => { } }); - it('should return InvalidTargetBucketForLogging when target bucket is owned by different account', - async () => { + it('should return InvalidTargetBucketForLogging when target bucket is owned by different account', async () => { const crossAccountConfig = { LoggingEnabled: { TargetBucket: otherAccountTargetBucket, @@ -239,10 +245,12 @@ describe('PUT bucket logging', () => { }; try { - await s3.send(new PutBucketLoggingCommand({ - Bucket: bucketName, - BucketLoggingStatus: crossAccountConfig, - })); + await s3.send( + new PutBucketLoggingCommand({ + Bucket: bucketName, + BucketLoggingStatus: crossAccountConfig, + }), + ); assert.fail('Expected InvalidTargetBucketForLogging error'); } catch (err) { assert.strictEqual(err.name, 'InvalidTargetBucketForLogging'); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js b/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js index e50d5c9c1b..5b8faee730 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketNotification.js @@ -1,7 +1,9 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - PutBucketNotificationConfigurationCommand } = require('@aws-sdk/client-s3'); + PutBucketNotificationConfigurationCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -52,7 +54,7 @@ describe('aws-sdk test put notification configuration', () => { }); describe('config rules', () => { - beforeEach(() => s3.send(new CreateBucketCommand({Bucket: bucket}))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -72,8 +74,7 @@ describe('aws-sdk test put notification configuration', () => { }); it('should put notification configuration on bucket with multiple events', async () => { - const params = getNotificationParams( - ['s3:ObjectCreated:*', 's3:ObjectRemoved:*']); + const params = getNotificationParams(['s3:ObjectCreated:*', 's3:ObjectRemoved:*']); await s3.send(new PutBucketNotificationConfigurationCommand(params)); }); @@ -136,29 +137,35 @@ describe('aws-sdk test put notification configuration', () => { it(`should handle ${event} events based on lifecycle rules configuration`, done => { const params = getNotificationParams([event]); const shouldSucceed = config.supportedLifecycleRules.some(rule => rule.includes(supported)); - s3.send(new PutBucketNotificationConfigurationCommand(params)).then(() => { - if (shouldSucceed) { - done(); - } else { - done(new Error('Expected MalformedXML error but operation succeeded')); - } - }).catch(err => { - if (shouldSucceed) { - done(err); - } else { - checkError(err, 'MalformedXML', 400); - done(); - } - }); + s3.send(new PutBucketNotificationConfigurationCommand(params)) + .then(() => { + if (shouldSucceed) { + done(); + } else { + done(new Error('Expected MalformedXML error but operation succeeded')); + } + }) + .catch(err => { + if (shouldSucceed) { + done(err); + } else { + checkError(err, 'MalformedXML', 400); + done(); + } + }); }); }); }); }); describe('cross origin requests', () => { - beforeEach(() => s3.send(new CreateBucketCommand({ - Bucket: bucket, - }))); + beforeEach(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + ), + ); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -167,11 +174,13 @@ describe('aws-sdk test put notification configuration', () => { it: 'return valid error with invalid arn', param: getNotificationParams(null, 'invalidArn'), error: 'MalformedXML', - }, { + }, + { it: 'return valid error with unknown/unsupported destination', param: getNotificationParams(null, 'arn:scality:bucketnotif:::target100'), error: 'InvalidArgument', - }, { + }, + { it: 'save notification configuration with correct arn', param: getNotificationParams(), }, diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js b/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js index 460c42d87e..94757c3d48 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketObjectLock.js @@ -1,7 +1,9 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, - PutObjectLockConfigurationCommand } = require('@aws-sdk/client-s3'); + PutObjectLockConfigurationCommand, +} = require('@aws-sdk/client-s3'); const checkError = require('../../lib/utility/checkError'); const getConfig = require('../support/config'); @@ -51,7 +53,7 @@ describe('aws-sdk test put object lock configuration', () => { }); describe('on object lock disabled bucket', () => { - beforeEach(() => s3.send(new CreateBucketCommand({Bucket: bucket}))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucket }))); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -82,10 +84,14 @@ describe('aws-sdk test put object lock configuration', () => { }); describe('config rules', () => { - beforeEach(() => s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - }))); + beforeEach(() => + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ), + ); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js b/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js index 9edbb25fcc..37ff5b31b0 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketPolicy.js @@ -1,9 +1,6 @@ const assert = require('assert'); const { errors } = require('arsenal'); -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand, - PutBucketPolicyCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketPolicyCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -55,9 +52,7 @@ function generateRandomString(length) { const allowedCharacters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+=,.@ -/'; const allowedCharactersLength = allowedCharacters.length; - return [...Array(length)] - .map(() => allowedCharacters[~~(Math.random() * allowedCharactersLength)]) - .join(''); + return [...Array(length)].map(() => allowedCharacters[~~(Math.random() * allowedCharactersLength)]).join(''); } // Check for the expected error response code and status code. @@ -65,15 +60,20 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.name, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.name}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, + assert.strictEqual( + err.name, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.name}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, 'incorrect error status code: should be ' + - `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`); + `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`, + ); } } - describe('aws-sdk test put bucket policy', () => { let s3; let otherAccountS3; @@ -170,12 +170,18 @@ describe('aws-sdk test put bucket policy', () => { }); it('should allow bucket policy with pincipal arn less than 2048 characters', async () => { - const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` } }); // eslint-disable-line max-len + const params = getPolicyParams({ + key: 'Principal', + value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(150)}` }, + }); await s3.send(new PutBucketPolicyCommand(params)); }); it('should not allow bucket policy with pincipal arn more than 2048 characters', async () => { - const params = getPolicyParams({ key: 'Principal', value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` } }); // eslint-disable-line max-len + const params = getPolicyParams({ + key: 'Principal', + value: { AWS: `arn:aws:iam::767707094035:user/${generateRandomString(2020)}` }, + }); try { await s3.send(new PutBucketPolicyCommand(params)); throw new Error('Expected MalformedPolicy error'); @@ -186,7 +192,8 @@ describe('aws-sdk test put bucket policy', () => { it('should allow bucket policy with valid SourceIp condition', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100.0/24', }, @@ -197,7 +204,8 @@ describe('aws-sdk test put bucket policy', () => { it('should not allow bucket policy with invalid SourceIp format', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100', // Invalid IP format }, @@ -213,7 +221,8 @@ describe('aws-sdk test put bucket policy', () => { it('should allow bucket policy with valid s3:object-lock-remaining-retention-days condition', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 's3:object-lock-remaining-retention-days': '30', }, @@ -225,7 +234,8 @@ describe('aws-sdk test put bucket policy', () => { // yep, this is the expected behaviour it('should not reject policy with invalid s3:object-lock-remaining-retention-days value', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 's3:object-lock-remaining-retention-days': '-1', // Invalid value }, @@ -237,7 +247,8 @@ describe('aws-sdk test put bucket policy', () => { // this too ¯\_(ツ)_/¯ it('should not reject policy with a key starting with aws:', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 'aws:have-a-nice-day': 'blabla', // Invalid value }, @@ -248,7 +259,8 @@ describe('aws-sdk test put bucket policy', () => { it('should reject policy with a key that does not exist that does not start with aws:', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { NumericGreaterThanEquals: { 'have-a-nice-day': 'blabla', // Invalid value }, @@ -264,7 +276,8 @@ describe('aws-sdk test put bucket policy', () => { it('should enforce policies with both SourceIp and s3:object-lock conditions together', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100.0/24', }, @@ -278,7 +291,8 @@ describe('aws-sdk test put bucket policy', () => { it('should return error if a condition one of the condition values is invalid', async () => { const params = getPolicyParams({ - key: 'Condition', value: { + key: 'Condition', + value: { IpAddress: { 'aws:SourceIp': '192.168.100', }, diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js b/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js index c2d0a94ba6..48b5099e49 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketRateLimit.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - S3Client, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const { sendRateLimitRequest, skipIfRateLimitDisabled } = require('../rateLimit/tooling'); const { config } = require('../../../../../lib/Config'); @@ -39,8 +35,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should set the rate limit config', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); assert.ok(true); } catch (err) { assert.ifError(err); @@ -50,15 +50,22 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should update existing rate limit config', async () => { try { const initialConfig = { RequestsPerSecond: 100 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(initialConfig)); - - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(initialConfig), + ); + + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); // Verify the update - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, 200); } catch (err) { assert.ifError(err); @@ -67,8 +74,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return NoSuchBucket error when bucket does not exist', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${nonExistentBucket}/?rate-limit`, JSON.stringify(rateLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${nonExistentBucket}/?rate-limit`, + JSON.stringify(rateLimitConfig), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'NoSuchBucket'); } @@ -76,8 +87,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when RequestsPerSecond is negative', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(invalidConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(invalidConfig), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -85,8 +100,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when RequestsPerSecond is not an integer', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(invalidConfigNotInteger)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(invalidConfigNotInteger), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -94,8 +113,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when RequestsPerSecond is missing', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(missingLimitConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(missingLimitConfig), + ); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -103,8 +126,7 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should return InvalidArgument error when request body is invalid JSON', async () => { try { - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, 'invalid json{'); + await sendRateLimitRequest('PUT', '127.0.0.1:8000', `/${bucket}/?rate-limit`, 'invalid json{'); } catch (err) { assert.strictEqual(err.Error.Code[0], 'InvalidArgument'); } @@ -113,19 +135,16 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should allow zero as a valid RequestsPerSecond value', async () => { try { const zeroConfig = { RequestsPerSecond: 0 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(zeroConfig)); + await sendRateLimitRequest('PUT', '127.0.0.1:8000', `/${bucket}/?rate-limit`, JSON.stringify(zeroConfig)); - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); - assert.deepStrictEqual(data, { RequestsPerSecond: { Limit: 0 } }); + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); + assert.deepStrictEqual(data, { RequestsPerSecond: { Limit: 0 } }); } catch (err) { assert.ifError(err); } }); describe('validation against node and worker count', () => { - const nodes = config.rateLimiting?.nodes || 1; const workers = config.clusters || 1; const minLimit = nodes * workers; @@ -138,8 +157,12 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { let error; try { const invalidConfig = { RequestsPerSecond: minLimit - 1 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(invalidConfig)); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(invalidConfig), + ); } catch (err) { error = err; } finally { @@ -151,11 +174,14 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should accept limits equal to (nodes x workers)', async () => { try { const validConfig = { RequestsPerSecond: minLimit }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(validConfig)); - - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(validConfig), + ); + + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, minLimit); } catch (err) { assert.ifError(err); @@ -165,11 +191,14 @@ skipIfRateLimitDisabled('Test put bucket rate limit', () => { it('should accept limits greater than (nodes x workers)', async () => { try { const validConfig = { RequestsPerSecond: minLimit + 1000 }; - await sendRateLimitRequest('PUT', '127.0.0.1:8000', - `/${bucket}/?rate-limit`, JSON.stringify(validConfig)); - - const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', - `/${bucket}/?rate-limit`); + await sendRateLimitRequest( + 'PUT', + '127.0.0.1:8000', + `/${bucket}/?rate-limit`, + JSON.stringify(validConfig), + ); + + const data = await sendRateLimitRequest('GET', '127.0.0.1:8000', `/${bucket}/?rate-limit`); assert.strictEqual(data.RequestsPerSecond.Limit, minLimit + 1000); } catch (err) { assert.ifError(err); diff --git a/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js b/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js index 35d26e1d15..d682355e65 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js +++ b/tests/functional/aws-node-sdk/test/bucket/putBucketTagging.js @@ -1,9 +1,11 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketTaggingCommand, - GetBucketTaggingCommand } = require('@aws-sdk/client-s3'); + GetBucketTaggingCommand, +} = require('@aws-sdk/client-s3'); const assertError = require('../../../../utilities/bucketTagging-util'); const getConfig = require('../support/config'); @@ -52,7 +54,8 @@ const validEmptyTagging = { const taggingKeyNotValid = { TagSet: [ { - Key: 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + + Key: + 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'astringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', Value: 'string', @@ -72,7 +75,8 @@ const taggingValueNotValid = { }, { Key: 'string', - Value: 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaa' + + Value: + 'stringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaastringaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + @@ -97,11 +101,13 @@ describe('aws-sdk test put bucket tagging', () => { it('should not add tag if tagKey not unique', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: taggingNotUnique, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: taggingNotUnique, + Bucket: bucket, + }), + ); throw new Error('Expected InvalidTag error'); } catch (err) { assertError(err, 'InvalidTag'); @@ -110,11 +116,13 @@ describe('aws-sdk test put bucket tagging', () => { it('should not add tag if tagKey not valid', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: taggingKeyNotValid, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: taggingKeyNotValid, + Bucket: bucket, + }), + ); throw new Error('Expected InvalidTag error'); } catch (err) { assertError(err, 'InvalidTag'); @@ -123,11 +131,13 @@ describe('aws-sdk test put bucket tagging', () => { it('should not add tag if tagValue not valid', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: taggingValueNotValid, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: taggingValueNotValid, + Bucket: bucket, + }), + ); throw new Error('Expected InvalidTag error'); } catch (err) { assertError(err, 'InvalidTag'); @@ -136,42 +146,54 @@ describe('aws-sdk test put bucket tagging', () => { it('should add tag', async () => { // Put bucket tagging - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validTagging, - Bucket: bucket, - })); - const res = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validTagging, + Bucket: bucket, + }), + ); + const res = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); assert.deepStrictEqual(res.TagSet, validTagging.TagSet); }); it('should be able to put single tag', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validSingleTagging, - Bucket: bucket, - })); - const res = await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket, - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validSingleTagging, + Bucket: bucket, + }), + ); + const res = await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); assert.deepStrictEqual(res.TagSet, validSingleTagging.TagSet); }); it('should be able to put empty tag array', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validEmptyTagging, - Bucket: bucket, - })); - try { - await s3.send(new GetBucketTaggingCommand({ + await s3.send( + new PutBucketTaggingCommand({ AccountId: s3.AccountId, + Tagging: validEmptyTagging, Bucket: bucket, - })); + }), + ); + try { + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -180,12 +202,14 @@ describe('aws-sdk test put bucket tagging', () => { it('should return accessDenied if expected bucket owner does not match', async () => { try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validEmptyTagging, - Bucket: bucket, - ExpectedBucketOwner: '944690102203' - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validEmptyTagging, + Bucket: bucket, + ExpectedBucketOwner: '944690102203', + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { assertError(err, 'AccessDenied'); @@ -193,17 +217,21 @@ describe('aws-sdk test put bucket tagging', () => { }); it('should not return accessDenied if expected bucket owner matches', async () => { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: validEmptyTagging, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: validEmptyTagging, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); try { - await s3.send(new GetBucketTaggingCommand({ - AccountId: s3.AccountId, - Bucket: bucket - })); + await s3.send( + new GetBucketTaggingCommand({ + AccountId: s3.AccountId, + Bucket: bucket, + }), + ); throw new Error('Expected NoSuchTagSet error'); } catch (err) { assertError(err, 'NoSuchTagSet'); @@ -217,12 +245,14 @@ describe('aws-sdk test put bucket tagging', () => { Value: `value_${index}`, })), }; - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: tags, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: tags, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); }); it('should not put more than 50 tags', async () => { @@ -233,12 +263,14 @@ describe('aws-sdk test put bucket tagging', () => { })), }; try { - await s3.send(new PutBucketTaggingCommand({ - AccountId: s3.AccountId, - Tagging: tags, - Bucket: bucket, - ExpectedBucketOwner: s3.AccountId - })); + await s3.send( + new PutBucketTaggingCommand({ + AccountId: s3.AccountId, + Tagging: tags, + Bucket: bucket, + ExpectedBucketOwner: s3.AccountId, + }), + ); throw new Error('Expected BadRequest error'); } catch (err) { assertError(err, 'BadRequest'); diff --git a/tests/functional/aws-node-sdk/test/bucket/putCors.js b/tests/functional/aws-node-sdk/test/bucket/putCors.js index 99cf6fe346..26247eea10 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putCors.js +++ b/tests/functional/aws-node-sdk/test/bucket/putCors.js @@ -1,25 +1,23 @@ const assert = require('assert'); -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand, - PutBucketCorsCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketCorsCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const getConfig = require('../support/config'); const bucketName = 'testcorsbucket'; -const sampleCors = { CORSRules: [ - { AllowedMethods: ['PUT', 'POST', 'DELETE'], - AllowedOrigins: ['http://www.example.com'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000, - ExposeHeaders: ['x-amz-server-side-encryption'] }, - { AllowedMethods: ['GET'], - AllowedOrigins: ['*'], - AllowedHeaders: ['*'], - MaxAgeSeconds: 3000 }, -] }; +const sampleCors = { + CORSRules: [ + { + AllowedMethods: ['PUT', 'POST', 'DELETE'], + AllowedOrigins: ['http://www.example.com'], + AllowedHeaders: ['*'], + MaxAgeSeconds: 3000, + ExposeHeaders: ['x-amz-server-side-encryption'], + }, + { AllowedMethods: ['GET'], AllowedOrigins: ['*'], AllowedHeaders: ['*'], MaxAgeSeconds: 3000 }, + ], +}; function _corsTemplate(params) { const sampleRule = { @@ -29,12 +27,11 @@ function _corsTemplate(params) { MaxAgeSeconds: 3000, ExposeHeaders: ['x-amz-server-side-encryption'], }; - ['AllowedMethods', 'AllowedOrigins', 'AllowedHeaders', 'MaxAgeSeconds', - 'ExposeHeaders'].forEach(prop => { - if (params[prop]) { - sampleRule[prop] = params[prop]; - } - }); + ['AllowedMethods', 'AllowedOrigins', 'AllowedHeaders', 'MaxAgeSeconds', 'ExposeHeaders'].forEach(prop => { + if (params[prop]) { + sampleRule[prop] = params[prop]; + } + }); return { CORSRules: [sampleRule] }; } @@ -45,10 +42,12 @@ describe('PUT bucket cors', () => { async function _testPutBucketCors(rules, statusCode, errMsg) { try { - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: rules - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: rules, + }), + ); throw new Error('Expected error but found none'); } catch (err) { assert.strictEqual(err.name, errMsg); @@ -56,15 +55,17 @@ describe('PUT bucket cors', () => { } } - beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucketName }))); it('should put a bucket cors successfully', async () => { - await s3.send(new PutBucketCorsCommand({ - Bucket: bucketName, - CORSConfiguration: sampleCors - })); + await s3.send( + new PutBucketCorsCommand({ + Bucket: bucketName, + CORSConfiguration: sampleCors, + }), + ); }); it('should return InvalidRequest if more than 100 rules', async () => { @@ -87,10 +88,8 @@ describe('PUT bucket cors', () => { await _testPutBucketCors(testCors, 400, 'MalformedXML'); }); - it('should return InvalidRequest if more than one asterisk in ' + - 'AllowedOrigin', async () => { - const testCors = - _corsTemplate({ AllowedOrigins: ['http://*.*.com'] }); + it('should return InvalidRequest if more than one asterisk in ' + 'AllowedOrigin', async () => { + const testCors = _corsTemplate({ AllowedOrigins: ['http://*.*.com'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); @@ -99,29 +98,28 @@ describe('PUT bucket cors', () => { await _testPutBucketCors(testCors, 400, 'MalformedXML'); }); - it('should return InvalidRequest if AllowedMethod is not a valid ' + - 'method', async () => { + it('should return InvalidRequest if AllowedMethod is not a valid ' + 'method', async () => { const testCors = _corsTemplate({ AllowedMethods: ['test'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); - it('should return InvalidRequest for lowercase value for ' + - 'AllowedMethod', async () => { + it('should return InvalidRequest for lowercase value for ' + 'AllowedMethod', async () => { const testCors = _corsTemplate({ AllowedMethods: ['put', 'get'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); - it('should return InvalidRequest if more than one asterisk in ' + - 'AllowedHeader', async () => { + it('should return InvalidRequest if more than one asterisk in ' + 'AllowedHeader', async () => { const testCors = _corsTemplate({ AllowedHeaders: ['*-amz-*'] }); await _testPutBucketCors(testCors, 400, 'InvalidRequest'); }); - it('should return InvalidRequest if ExposeHeader has character ' + - 'that is not dash or alphanumeric', async () => { - const testCors = _corsTemplate({ ExposeHeaders: ['test header'] }); - await _testPutBucketCors(testCors, 400, 'InvalidRequest'); - }); + it( + 'should return InvalidRequest if ExposeHeader has character ' + 'that is not dash or alphanumeric', + async () => { + const testCors = _corsTemplate({ ExposeHeaders: ['test header'] }); + await _testPutBucketCors(testCors, 400, 'InvalidRequest'); + }, + ); it('should return InvalidRequest if ExposeHeader has wildcard', async () => { const testCors = _corsTemplate({ ExposeHeaders: ['x-amz-*'] }); diff --git a/tests/functional/aws-node-sdk/test/bucket/putWebsite.js b/tests/functional/aws-node-sdk/test/bucket/putWebsite.js index 0d2e8181a3..adc53c2de4 100644 --- a/tests/functional/aws-node-sdk/test/bucket/putWebsite.js +++ b/tests/functional/aws-node-sdk/test/bucket/putWebsite.js @@ -1,8 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutBucketWebsiteCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutBucketWebsiteCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -16,18 +13,17 @@ describe('PUT bucket website', () => { const s3 = bucketUtil.s3; function _testPutBucketWebsite(config, statusCode, errMsg, cb) { - s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, - WebsiteConfiguration: config })) - .then(() => { - cb(new Error('Expected err but found none')); - }) - .catch(err => { - assert.strictEqual(err.name, errMsg); - assert.strictEqual(err.$metadata.httpStatusCode, statusCode); - cb(); - }); + s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, WebsiteConfiguration: config })) + .then(() => { + cb(new Error('Expected err but found none')); + }) + .catch(err => { + assert.strictEqual(err.name, errMsg); + assert.strictEqual(err.$metadata.httpStatusCode, statusCode); + cb(); + }); } - beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); + beforeEach(() => s3.send(new CreateBucketCommand({ Bucket: bucketName }))); afterEach(async () => { await bucketUtil.empty(bucketName); @@ -36,58 +32,59 @@ describe('PUT bucket website', () => { it('should put a bucket website successfully', () => { const config = new WebsiteConfigTester('index.html'); - s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, - WebsiteConfiguration: config })); + s3.send(new PutBucketWebsiteCommand({ Bucket: bucketName, WebsiteConfiguration: config })); }); - it('should return InvalidArgument if IndexDocument or ' + - 'RedirectAllRequestsTo is not provided', done => { + it('should return InvalidArgument if IndexDocument or ' + 'RedirectAllRequestsTo is not provided', done => { const config = new WebsiteConfigTester(); _testPutBucketWebsite(config, 400, 'InvalidArgument', done); }); - it('should return an InvalidRequest if both ' + - 'RedirectAllRequestsTo and IndexDocument are provided', done => { - const redirectAllTo = { - HostName: 'test', - Protocol: 'http', - }; - const config = new WebsiteConfigTester(null, null, - redirectAllTo); - config.addRoutingRule({ Protocol: 'http' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); + it( + 'should return an InvalidRequest if both ' + 'RedirectAllRequestsTo and IndexDocument are provided', + done => { + const redirectAllTo = { + HostName: 'test', + Protocol: 'http', + }; + const config = new WebsiteConfigTester(null, null, redirectAllTo); + config.addRoutingRule({ Protocol: 'http' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); it('should return InvalidArgument if index has slash', done => { const config = new WebsiteConfigTester('in/dex.html'); _testPutBucketWebsite(config, 400, 'InvalidArgument', done); }); - it('should return InvalidRequest if both ReplaceKeyWith and ' + - 'ReplaceKeyPrefixWith are present in same rule', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', - ReplaceKeyWith: 'test' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); - - it('should return InvalidRequest if both ReplaceKeyWith and ' + - 'ReplaceKeyPrefixWith are present in same rule', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', - ReplaceKeyWith: 'test' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); - - it('should return InvalidRequest if Redirect Protocol is ' + - 'not http or https', done => { + it( + 'should return InvalidRequest if both ReplaceKeyWith and ' + + 'ReplaceKeyPrefixWith are present in same rule', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', ReplaceKeyWith: 'test' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); + + it( + 'should return InvalidRequest if both ReplaceKeyWith and ' + + 'ReplaceKeyPrefixWith are present in same rule', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ ReplaceKeyPrefixWith: 'test', ReplaceKeyWith: 'test' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); + + it('should return InvalidRequest if Redirect Protocol is ' + 'not http or https', done => { const config = new WebsiteConfigTester('index.html'); config.addRoutingRule({ Protocol: 'notvalidprotocol' }); _testPutBucketWebsite(config, 400, 'InvalidRequest', done); }); - it('should return InvalidRequest if RedirectAllRequestsTo Protocol ' + - 'is not http or https', done => { + it('should return InvalidRequest if RedirectAllRequestsTo Protocol ' + 'is not http or https', done => { const redirectAllTo = { HostName: 'test', Protocol: 'notvalidprotocol', @@ -96,36 +93,47 @@ describe('PUT bucket website', () => { _testPutBucketWebsite(config, 400, 'InvalidRequest', done); }); - it('should return MalformedXML if Redirect HttpRedirectCode ' + - 'is a string that does not contains a number', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HttpRedirectCode: 'notvalidhttpcode' }); - _testPutBucketWebsite(config, 400, 'MalformedXML', done); - }); - - it('should return InvalidRequest if Redirect HttpRedirectCode ' + - 'is not a valid http redirect code (3XX excepting 300)', done => { - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HttpRedirectCode: '400' }); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); - - it('should return InvalidRequest if Condition ' + - 'HttpErrorCodeReturnedEquals is a string that does ' + - ' not contain a number', done => { - const condition = { HttpErrorCodeReturnedEquals: 'notvalidcode' }; - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HostName: 'test' }, condition); - _testPutBucketWebsite(config, 400, 'MalformedXML', done); - }); - - it('should return InvalidRequest if Condition ' + - 'HttpErrorCodeReturnedEquals is not a valid http' + - 'error code (4XX or 5XX)', done => { - const condition = { HttpErrorCodeReturnedEquals: '300' }; - const config = new WebsiteConfigTester('index.html'); - config.addRoutingRule({ HostName: 'test' }, condition); - _testPutBucketWebsite(config, 400, 'InvalidRequest', done); - }); + it( + 'should return MalformedXML if Redirect HttpRedirectCode ' + 'is a string that does not contains a number', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HttpRedirectCode: 'notvalidhttpcode' }); + _testPutBucketWebsite(config, 400, 'MalformedXML', done); + }, + ); + + it( + 'should return InvalidRequest if Redirect HttpRedirectCode ' + + 'is not a valid http redirect code (3XX excepting 300)', + done => { + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HttpRedirectCode: '400' }); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); + + it( + 'should return InvalidRequest if Condition ' + + 'HttpErrorCodeReturnedEquals is a string that does ' + + ' not contain a number', + done => { + const condition = { HttpErrorCodeReturnedEquals: 'notvalidcode' }; + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HostName: 'test' }, condition); + _testPutBucketWebsite(config, 400, 'MalformedXML', done); + }, + ); + + it( + 'should return InvalidRequest if Condition ' + + 'HttpErrorCodeReturnedEquals is not a valid http' + + 'error code (4XX or 5XX)', + done => { + const condition = { HttpErrorCodeReturnedEquals: '300' }; + const config = new WebsiteConfigTester('index.html'); + config.addRoutingRule({ HostName: 'test' }, condition); + _testPutBucketWebsite(config, 400, 'InvalidRequest', done); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/skipScan.js b/tests/functional/aws-node-sdk/test/bucket/skipScan.js index e846b8d713..fb4f13555d 100644 --- a/tests/functional/aws-node-sdk/test/bucket/skipScan.js +++ b/tests/functional/aws-node-sdk/test/bucket/skipScan.js @@ -1,9 +1,11 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutObjectCommand, ListObjectsCommand, - DeleteObjectCommand } = require('@aws-sdk/client-s3'); + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -45,7 +47,7 @@ describe('Skip scan cases tests', () => { let s3; before(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); - s3 = new S3Client(config); + s3 = new S3Client(config); await s3.send(new CreateBucketCommand({ Bucket })); const x = 120; const promises = []; @@ -54,30 +56,27 @@ describe('Skip scan cases tests', () => { const o = {}; o.Bucket = Bucket; // eslint-disable-next-line - o.Key = String.fromCharCode(65 + n / x) + - '/' + n % x; + o.Key = String.fromCharCode(65 + n / x) + '/' + (n % x); o.Body = ''; await s3.send(new PutObjectCommand(o)); }; promises.push(putObjectPromise); - } + } for (let i = 0; i < promises.length; i += 10) { const batch = promises.slice(i, i + 10); await Promise.all(batch.map(fn => fn())); } }); - + after(async () => { const data = await s3.send(new ListObjectsCommand({ Bucket })); - const deletePromises = data.Contents.map(o => - s3.send(new DeleteObjectCommand({ Bucket, Key: o.Key })) - ); + const deletePromises = data.Contents.map(o => s3.send(new DeleteObjectCommand({ Bucket, Key: o.Key }))); await Promise.all(deletePromises); await s3.send(new DeleteBucketCommand({ Bucket })); }); - + it('should find all common prefixes in one shot', async () => { - const { $metadata , ...data } = await s3.send(new ListObjectsCommand({ Bucket, Delimiter: '/' })); + const { $metadata, ...data } = await s3.send(new ListObjectsCommand({ Bucket, Delimiter: '/' })); cutAttributes(data); assert.deepStrictEqual(data, { IsTruncated: false, @@ -86,13 +85,7 @@ describe('Skip scan cases tests', () => { Name: Bucket, Prefix: '', MaxKeys: 1000, - CommonPrefixes: [ - 'A/', - 'B/', - 'C/', - 'D/', - 'E/', - ], + CommonPrefixes: ['A/', 'B/', 'C/', 'D/', 'E/'], }); assert.strictEqual($metadata.httpStatusCode, 200); }); diff --git a/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js b/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js index a9b37b792a..bb4e3dd643 100644 --- a/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js +++ b/tests/functional/aws-node-sdk/test/bucket/testBucketStress.js @@ -1,8 +1,10 @@ -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutObjectCommand, - DeleteObjectCommand } = require('@aws-sdk/client-s3'); + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); diff --git a/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js b/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js index 09ea95d26d..56ed756abe 100644 --- a/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js +++ b/tests/functional/aws-node-sdk/test/bucket/testBucketVersioning.js @@ -1,16 +1,17 @@ const assert = require('assert'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketVersioningCommand, - GetBucketVersioningCommand } = require('@aws-sdk/client-s3'); + GetBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); const bucket = `versioning-bucket-${Date.now()}`; const config = getConfig('default', { signatureVersion: 'v4' }); -const configReplication = getConfig('replication', - { signatureVersion: 'v4' }); +const configReplication = getConfig('replication', { signatureVersion: 'v4' }); const s3 = new S3Client(config); describe('aws-node-sdk test bucket versioning', function testSuite() { this.timeout(60000); @@ -33,8 +34,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { throw new Error('accepted empty versioning configuration'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); } }); @@ -57,8 +57,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { throw new Error('accepted empty versioning configuration'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); } }); @@ -74,7 +73,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { Bucket: bucket, VersioningConfiguration: { MFADelete: 'fun', - Status: 'let\'s do it', + Status: "let's do it", }, }; try { @@ -82,8 +81,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { throw new Error('accepted empty versioning configuration'); } catch (error) { assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); } }); @@ -142,8 +140,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { // S3C doesn't support service account. There is no cross account access for replication account. // (canonicalId looking like http://acs.zenko.io/accounts/service/replication) const itSkipS3C = process.env.S3_END_TO_END ? it.skip : it; - itSkipS3C('should accept valid versioning configuration if user is a ' + - 'replication user', async () => { + itSkipS3C('should accept valid versioning configuration if user is a ' + 'replication user', async () => { const params = { Bucket: bucket, VersioningConfiguration: { @@ -160,26 +157,31 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }); }); - describe('bucket versioning for ingestion buckets', () => { const Bucket = `ingestion-bucket-${Date.now()}`; - before(() => s3.send(new CreateBucketCommand({ - Bucket, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-2:ingest', - }, - }))); + before(() => + s3.send( + new CreateBucketCommand({ + Bucket, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + }), + ), + ); after(() => s3.send(new DeleteBucketCommand({ Bucket }))); it('should not allow suspending versioning for ingestion buckets', async () => { try { - await s3.send(new PutBucketVersioningCommand({ - Bucket, - VersioningConfiguration: { - Status: 'Suspended' - } - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket, + VersioningConfiguration: { + Status: 'Suspended', + }, + }), + ); throw new Error('Expected error but got success'); } catch (err) { assert.strictEqual(err.name, 'InvalidBucketState'); @@ -193,10 +195,12 @@ describe('aws-node-sdk test bucket versioning with object lock', () => { before(async () => { const config = getConfig('default', { signatureVersion: 'v4' }); s3ObjectLock = new S3Client(config); - await s3ObjectLock.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3ObjectLock.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); }); after(() => s3ObjectLock.send(new DeleteBucketCommand({ Bucket: bucket }))); @@ -216,4 +220,3 @@ describe('aws-node-sdk test bucket versioning with object lock', () => { } }); }); - diff --git a/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js b/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js index fc14716652..68e622269d 100644 --- a/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js +++ b/tests/functional/aws-node-sdk/test/bucket/updateBucketQuota.js @@ -1,6 +1,4 @@ -const { S3Client, - CreateBucketCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); +const { S3Client, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const assert = require('assert'); const getConfig = require('../support/config'); @@ -25,19 +23,21 @@ describe('Test update bucket quota', () => { afterEach(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - it('should update the quota, using json parsing by default', () => sendRequest('PUT', - '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota))); + it('should update the quota, using json parsing by default', () => + sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota))); it('should update quota with explicit JSON content-type', async () => { - await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, - JSON.stringify(quota), null, new Date(), { 'Content-Type': 'application/json' }); + await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(quota), null, new Date(), { + 'Content-Type': 'application/json', + }); }); it('should update quota with XML format', async () => { try { const xmlQuota = '3000'; - await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, - xmlQuota, null, new Date(), { 'Content-Type': 'application/xml' }); + await sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, xmlQuota, null, new Date(), { + 'Content-Type': 'application/xml', + }); assert.ok(true); } catch (err) { assert.fail(`Expected no error, but got ${err}`); @@ -70,6 +70,6 @@ describe('Test update bucket quota', () => { } }); - it('should accept large quota', () => sendRequest('PUT', - '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(largeQuota))); + it('should accept large quota', () => + sendRequest('PUT', '127.0.0.1:8000', `/${bucket}/?quota=true`, JSON.stringify(largeQuota))); }); diff --git a/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js b/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js index 683857f1d7..0536b160a8 100644 --- a/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js +++ b/tests/functional/aws-node-sdk/test/legacy/authV2QueryTests.js @@ -19,10 +19,7 @@ const provideRawOutputAsync = util.promisify(provideRawOutput); const random = Math.round(Math.random() * 100).toString(); const bucket = `mybucket-${random}`; const almostOutsideTime = 99990; -const itSkipAWS = process.env.AWS_ON_AIR - ? it.skip - : it; - +const itSkipAWS = process.env.AWS_ON_AIR ? it.skip : it; function diff(putFile, receivedFile, done) { process.stdout.write(`diff ${putFile} ${receivedFile}\n`); @@ -51,13 +48,11 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { // AWS allows an expiry further in the future // 604810 seconds is higher that the Expires time limit: 604800 seconds // ( seven days) - itSkipAWS('should return an error code if expires header is too far ' + - 'in the future', async () => { - + itSkipAWS('should return an error code if expires header is too far ' + 'in the future', async () => { // First, get a valid signed URL with maximum allowed expiry const command = new CreateBucketCommand({ Bucket: bucket }); const validUrl = await getSignedUrl(s3, command, { expiresIn: 604800 }); // Exactly 7 days - + // Manually modify the URL to have a longer expiry const urlObj = new URL(validUrl); const futureExpiry = Math.floor(Date.now() / 1000) + 604810; // 10 seconds more than limit @@ -82,12 +77,10 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { assert.strictEqual(httpCode, '200 OK'); }); - it('should put an object', async () => { const command = new PutObjectCommand({ Bucket: bucket, Key: 'key' }); const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); @@ -97,19 +90,17 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { // This will also test that query params that contain "x-amz-" // are being added to the canonical headers list in our string // to sign. - const command = new PutObjectCommand({ - Bucket: bucket, + const command = new PutObjectCommand({ + Bucket: bucket, Key: 'key', - ACL: 'public-read', - StorageClass: 'STANDARD' + ACL: 'public-read', + StorageClass: 'STANDARD', }); const url = await getSignedUrl(s3, command); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); - it('should get an object', async () => { const command = new GetObjectCommand({ Bucket: bucket, Key: 'key' }); const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime }); @@ -130,7 +121,6 @@ describe('aws-node-sdk v2auth query tests', function testSuite() { assert.strictEqual(httpCode, '204 NO CONTENT'); }); - it('should delete a bucket', async () => { const command = new DeleteBucketCommand({ Bucket: bucket }); const url = await getSignedUrl(s3, command, { expiresIn: almostOutsideTime }); diff --git a/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js b/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js index 23df1352cd..3142d8148b 100644 --- a/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js +++ b/tests/functional/aws-node-sdk/test/legacy/authV4QueryTests.js @@ -55,11 +55,7 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { it('should create a bucket', async () => { const params = { Bucket: bucket }; - const url = await getSignedUrl( - s3, - new CreateBucketCommand(params), - { expiresIn: 900 } - ); + const url = await getSignedUrl(s3, new CreateBucketCommand(params), { expiresIn: 900 }); const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url]); assert.strictEqual(httpCode, '200 OK'); }); @@ -69,16 +65,14 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { const { httpCode, rawOutput } = await provideRawOutputAsync(['-verbose', url]); assert.strictEqual(httpCode, '200 OK'); const xml = await parseStringAsync(rawOutput.stdout); - const bucketNames = xml.ListAllMyBucketsResult - .Buckets[0].Bucket.map(item => item.Name[0]); + const bucketNames = xml.ListAllMyBucketsResult.Buckets[0].Bucket.map(item => item.Name[0]); assert(bucketNames.indexOf(bucket) > -1); }); it('should put an object', async () => { const params = { Bucket: bucket, Key: 'key' }; const url = await getSignedUrl(s3, new PutObjectCommand(params), { expiresIn: 900 }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); @@ -91,18 +85,17 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { ContentType: 'text/plain', }; const url = await getSignedUrl(s3, new PutObjectCommand(params), { expiresIn: 900 }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); it('should put an object with native characters', async () => { - const Key = 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + - 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; + const Key = + 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + + 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; const params = { Bucket: bucket, Key }; const url = await getSignedUrl(s3, new PutObjectCommand(params), { expiresIn: 900 }); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', 'uploadFile']); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', 'uploadFile']); assert.strictEqual(httpCode, '200 OK'); }); @@ -153,8 +146,9 @@ describe('aws-node-sdk v4auth query tests', function testSuite() { }); it('should delete an object with native characters', async () => { - const Key = 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + - 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; + const Key = + 'key-pâtisserie-中文-español-English-हिन्दी-العربية-' + + 'português-বাংলা-русский-日本語-ਪੰਜਾਬੀ-한국어-தமிழ்'; const params = { Bucket: bucket, Key }; const url = await getSignedUrl(s3, new DeleteObjectCommand(params), { expiresIn: 900 }); const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'DELETE', url]); diff --git a/tests/functional/aws-node-sdk/test/legacy/tests.js b/tests/functional/aws-node-sdk/test/legacy/tests.js index fd704f9014..ed9e4535c3 100644 --- a/tests/functional/aws-node-sdk/test/legacy/tests.js +++ b/tests/functional/aws-node-sdk/test/legacy/tests.js @@ -156,21 +156,24 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { assert.strictEqual(data.StorageClass, 'STANDARD'); }); - it('should return an error if do not provide correct ' + - // completempu test - 'xml when completing a multipart upload', async () => { - const params = { - Bucket: bucket, - Key: 'toComplete', - UploadId: multipartUploadData.secondUploadId, - }; - try { - await s3.send(new CompleteMultipartUploadCommand(params)); - throw new Error('Expected MalformedXML error'); - } catch (err) { - assert.strictEqual(err.Code, 'MalformedXML'); - } - }); + it( + 'should return an error if do not provide correct ' + + // completempu test + 'xml when completing a multipart upload', + async () => { + const params = { + Bucket: bucket, + Key: 'toComplete', + UploadId: multipartUploadData.secondUploadId, + }; + try { + await s3.send(new CompleteMultipartUploadCommand(params)); + throw new Error('Expected MalformedXML error'); + } catch (err) { + assert.strictEqual(err.Code, 'MalformedXML'); + } + }, + ); // completempu test it('should complete a multipart upload', async () => { @@ -214,8 +217,8 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { }); const mpuRangeGetTests = [ - { it: 'should get a range from the first part of an object ' + - 'put by multipart upload', + { + it: 'should get a range from the first part of an object ' + 'put by multipart upload', range: 'bytes=0-9', contentLength: 10, contentRange: 'bytes 0-9/10485760', @@ -224,8 +227,8 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { // first part should just contain 0 expectedBuff: Buffer.alloc(10, 0), }, - { it: 'should get a range from the second part of an object ' + - 'put by multipart upload', + { + it: 'should get a range from the second part of an object ' + 'put by multipart upload', // The completed MPU byte count starts at 0, so the first part ends // at byte 5242879 and the second part begins at byte 5242880 range: 'bytes=5242880-5242889', @@ -234,8 +237,8 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { // A range from the second part should just contain 1 expectedBuff: Buffer.alloc(10, 1), }, - { it: 'should get a range that spans both parts of an object put ' + - 'by multipart upload', + { + it: 'should get a range that spans both parts of an object put ' + 'by multipart upload', range: 'bytes=5242875-5242884', contentLength: 10, contentRange: 'bytes 5242875-5242884/10485760', @@ -243,9 +246,11 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { // of 0 and 5 bytes of 1 expectedBuff: Buffer.allocUnsafe(10).fill(0, 0, 5).fill(1, 5, 10), }, - { it: 'should get a range from the second part of an object put by ' + - 'multipart upload and include the end even if the range ' + - 'requested goes beyond the actual object end', + { + it: + 'should get a range from the second part of an object put by ' + + 'multipart upload and include the end even if the range ' + + 'requested goes beyond the actual object end', // End is actually 10485759 since size is 10485760 range: 'bytes=10485750-10485790', contentLength: 10, @@ -303,8 +308,7 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { assert.ok(data); }); - it('should return InvalidRange if the range of the resource does ' + - 'not cover the byte range', async () => { + it('should return InvalidRange if the range of the resource does ' + 'not cover the byte range', async () => { const params = { Bucket: bucket, Key: 'normalput', @@ -333,8 +337,7 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { }); testsRangeOnEmptyFile.forEach(test => { const validText = test.valid ? 'InvalidRange error' : 'empty file'; - it(`should return ${validText} if get range ${test.range} on ` + - 'empty object', async () => { + it(`should return ${validText} if get range ${test.range} on ` + 'empty object', async () => { const getParams = { Bucket: bucketEmptyObj, Key: 'emptyobj', @@ -363,28 +366,30 @@ describe('aws-node-sdk test suite as registered user', function testSuite() { }); const regularObjectRangeGetTests = [ - { it: 'should get a range for an object put without MPU', + { + it: 'should get a range for an object put without MPU', range: 'bytes=10-99', contentLength: 90, contentRange: 'bytes 10-99/200', // Buffer.fill(value, offset, end) expectedBuff: Buffer.allocUnsafe(90).fill(0, 0, 40).fill(1, 40), }, - { it: 'should get a range for an object using only an end ' + - 'offset in the request', + { + it: 'should get a range for an object using only an end ' + 'offset in the request', range: 'bytes=-10', contentLength: 10, contentRange: 'bytes 190-199/200', expectedBuff: Buffer.alloc(10, 1), }, - { it: 'should get a range for an object using only a start offset ' + - 'in the request', + { + it: 'should get a range for an object using only a start offset ' + 'in the request', range: 'bytes=190-', contentLength: 10, contentRange: 'bytes 190-199/200', expectedBuff: Buffer.alloc(10, 1), }, - { it: 'should get full object if range header is invalid', + { + it: 'should get full object if range header is invalid', range: 'bytes=-', contentLength: 200, // Since range header is invalid full object should be returned diff --git a/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js b/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js index d97876bc92..4166e01649 100644 --- a/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js +++ b/tests/functional/aws-node-sdk/test/mdSearch/basicSearch.js @@ -16,136 +16,130 @@ const updatedUserMetadata = { food: 'cake' }; runIfMongo('Basic search', () => { const bucketName = `basicsearchmebucket${Date.now()}`; - + before(async () => { await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Metadata: userMetadata, - Tagging: objectTagData, - })); - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: hiddenKey, - Tagging: hiddenTagData, - })); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Metadata: userMetadata, + Tagging: objectTagData, + }), + ); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: hiddenKey, + Tagging: hiddenTagData, + }), + ); }); after(async () => { - await s3Client.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { Key: objectKey }, - { Key: hiddenKey }, - ], - }, - })); + await s3Client.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [{ Key: objectKey }, { Key: hiddenKey }], + }, + }), + ); await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })); }); it('should list object with searched for system metadata', done => { const encodedSearch = encodeURIComponent(`key="${objectKey}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should list object with regex searched for system metadata', done => { const encodedSearch = encodeURIComponent('key LIKE "find.*"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); - it('should list object with regex searched for system metadata with flags', - done => { + it('should list object with regex searched for system metadata with flags', done => { const encodedSearch = encodeURIComponent('key LIKE "/FIND.*/i"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should return empty when no object match regex', done => { const encodedSearch = encodeURIComponent('key LIKE "/NOTFOUND.*/i"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, null, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, null, done); }); it('should list object with searched for user metadata', done => { - const encodedSearch = - encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + const encodedSearch = encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should list object with searched for tag metadata', done => { - const encodedSearch = - encodeURIComponent('tags.item-type="main"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); + const encodedSearch = encodeURIComponent('tags.item-type="main"'); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); }); it('should return empty listing when no object has user md', done => { - const encodedSearch = - encodeURIComponent('x-amz-meta-food="nosuchfood"'); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, null, done); + const encodedSearch = encodeURIComponent('x-amz-meta-food="nosuchfood"'); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, null, done); }); describe('search when overwrite object', () => { before(done => { - s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Metadata: updatedUserMetadata, - })) + s3Client + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Metadata: updatedUserMetadata, + }), + ) .then(() => done()) .catch(done); }); - it('should list object with searched for updated user metadata', - done => { - const encodedSearch = - encodeURIComponent('x-amz-meta-food' + - `="${updatedUserMetadata.food}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, objectKey, done); - }); + it('should list object with searched for updated user metadata', done => { + const encodedSearch = encodeURIComponent('x-amz-meta-food' + `="${updatedUserMetadata.food}"`); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, objectKey, done); + }); }); }); runIfMongo('Search when no objects in bucket', () => { const bucketName = `noobjectbucket${Date.now()}`; - + before(done => { - s3Client.send(new CreateBucketCommand({ Bucket: bucketName })) + s3Client + .send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); after(done => { - s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })) + s3Client + .send(new DeleteBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); it('should return empty listing when no objects in bucket', done => { const encodedSearch = encodeURIComponent(`key="${objectKey}"`); - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, null, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, null, done); }); }); runIfMongo('Invalid regular expression searches', () => { const bucketName = `badregex-${Date.now()}`; - + before(done => { - s3Client.send(new CreateBucketCommand({ Bucket: bucketName })) + s3Client + .send(new CreateBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); after(done => { - s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })) + s3Client + .send(new DeleteBucketCommand({ Bucket: bucketName })) .then(() => done()) .catch(done); }); @@ -156,7 +150,6 @@ runIfMongo('Invalid regular expression searches', () => { code: 'InvalidArgument', message: 'Invalid sql where clause sent as search query', }; - return runAndCheckSearch(s3Client, bucketName, - encodedSearch, false, testError, done); + return runAndCheckSearch(s3Client, bucketName, encodedSearch, false, testError, done); }); }); diff --git a/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js b/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js index a3b14b2c77..1e882d8431 100644 --- a/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js +++ b/tests/functional/aws-node-sdk/test/mdSearch/utils/helpers.js @@ -1,10 +1,6 @@ const assert = require('assert'); const async = require('async'); -const { - ListObjectsCommand, - ListObjectVersionsCommand, - DeleteObjectsCommand, -} = require('@aws-sdk/client-s3'); +const { ListObjectsCommand, ListObjectVersionsCommand, DeleteObjectsCommand } = require('@aws-sdk/client-s3'); function _deleteVersionList(s3Client, versionList, bucket, callback) { if (versionList === undefined || versionList.length === 0) { @@ -19,31 +15,30 @@ function _deleteVersionList(s3Client, versionList, bucket, callback) { })), }, }; - return s3Client.send(new DeleteObjectsCommand(params)) + return s3Client + .send(new DeleteObjectsCommand(params)) .then(() => callback()) .catch(callback); } const testUtils = {}; -testUtils.runIfMongo = process.env.S3METADATA === 'mongodb' ? - describe : describe.skip; +testUtils.runIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip; -testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions, - testResult, done) => { +testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions, testResult, done) => { const makeRequest = async () => { try { - const input = { + const input = { Bucket: bucketName, }; - + let command; if (listVersions) { command = new ListObjectVersionsCommand(input); } else { command = new ListObjectsCommand(input); } - + // Add middleware to inject the search query parameter // SDK v3 automatically encodes query parameters, so we decode first to avoid double-encoding command.middlewareStack.add( @@ -59,26 +54,30 @@ testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions // eslint-disable-next-line no-param-reassign args.request.query.versions = ''; } - + return next(args); }, { step: 'build', name: 'addSearchQuery', - } + }, ); - + const res = await s3Client.send(command); - + if (listVersions) { if (testResult) { assert.notStrictEqual(res.Versions[0].VersionId, undefined); if (Array.isArray(testResult)) { assert.strictEqual(res.Versions.length, testResult.length); - async.forEachOf(testResult, (expected, i, next) => { - assert.strictEqual(res.Versions[i].Key, expected); - next(); - }, done); + async.forEachOf( + testResult, + (expected, i, next) => { + assert.strictEqual(res.Versions[i].Key, expected); + next(); + }, + done, + ); } else { assert(res.Versions[0], 'should be Contents listed'); assert.strictEqual(res.Versions[0].Key, testResult); @@ -114,26 +113,29 @@ testUtils.runAndCheckSearch = (s3Client, bucketName, encodedSearch, listVersions }; testUtils.removeAllVersions = (s3Client, bucket, callback) => { - async.waterfall([ - cb => s3Client.send(new ListObjectVersionsCommand({ Bucket: bucket })) - .then(data => cb(null, data)) - .catch(cb), - (data, cb) => _deleteVersionList(s3Client, data.DeleteMarkers, bucket, - err => cb(err, data)), - (data, cb) => _deleteVersionList(s3Client, data.Versions, bucket, - err => cb(err, data)), - (data, cb) => { - if (data.IsTruncated) { - const params = { - Bucket: bucket, - KeyMarker: data.NextKeyMarker, - VersionIdMarker: data.NextVersionIdMarker, - }; - return testUtils.removeAllVersions(s3Client, params, cb); - } - return cb(); - }, - ], callback); + async.waterfall( + [ + cb => + s3Client + .send(new ListObjectVersionsCommand({ Bucket: bucket })) + .then(data => cb(null, data)) + .catch(cb), + (data, cb) => _deleteVersionList(s3Client, data.DeleteMarkers, bucket, err => cb(err, data)), + (data, cb) => _deleteVersionList(s3Client, data.Versions, bucket, err => cb(err, data)), + (data, cb) => { + if (data.IsTruncated) { + const params = { + Bucket: bucket, + KeyMarker: data.NextKeyMarker, + VersionIdMarker: data.NextVersionIdMarker, + }; + return testUtils.removeAllVersions(s3Client, params, cb); + } + return cb(); + }, + ], + callback, + ); }; module.exports = testUtils; diff --git a/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js b/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js index 904f3b5696..7e83b6f75d 100644 --- a/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js +++ b/tests/functional/aws-node-sdk/test/mdSearch/versionEnabledSearch.js @@ -6,8 +6,7 @@ const { } = require('@aws-sdk/client-s3'); const { promisify } = require('util'); const s3Client = require('./utils/s3SDK'); -const { runAndCheckSearch, removeAllVersions, runIfMongo } = - require('./utils/helpers'); +const { runAndCheckSearch, removeAllVersions, runIfMongo } = require('./utils/helpers'); const removeAllVersionsPromise = promisify(removeAllVersions); const userMetadata = { food: 'pizza' }; const updatedMetadata = { food: 'pineapple' }; @@ -21,48 +20,50 @@ runIfMongo('Search in version enabled bucket', () => { }; before(async () => { await s3Client.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3Client.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration - })); - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: masterKey, - Metadata: userMetadata - })); + await s3Client.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration, + }), + ); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: masterKey, + Metadata: userMetadata, + }), + ); }); after(async () => { - await removeAllVersionsPromise(s3Client, bucketName); - await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })); + await removeAllVersionsPromise(s3Client, bucketName); + await s3Client.send(new DeleteBucketCommand({ Bucket: bucketName })); }); it('should list just master object with searched for metadata by default', done => { - const encodedSearch = - encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); + const encodedSearch = encodeURIComponent(`x-amz-meta-food="${userMetadata.food}"`); runAndCheckSearch(s3Client, bucketName, encodedSearch, false, masterKey, done); }); describe('New version overwrite', () => { before(async () => { - await s3Client.send(new PutObjectCommand({ - Bucket: bucketName, - Key: masterKey, - Metadata: updatedMetadata - })); + await s3Client.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: masterKey, + Metadata: updatedMetadata, + }), + ); }); it('should list just master object with updated metadata by default', done => { - const encodedSearch = - encodeURIComponent(`x-amz-meta-food="${updatedMetadata.food}"`); + const encodedSearch = encodeURIComponent(`x-amz-meta-food="${updatedMetadata.food}"`); runAndCheckSearch(s3Client, bucketName, encodedSearch, false, masterKey, done); }); it('should list all object versions that met search query while specifying versions param', done => { - const encodedSearch = - encodeURIComponent('x-amz-meta-food LIKE "pi.*"'); - runAndCheckSearch(s3Client, bucketName, - encodedSearch, true, [masterKey, masterKey], done); + const encodedSearch = encodeURIComponent('x-amz-meta-food LIKE "pi.*"'); + runAndCheckSearch(s3Client, bucketName, encodedSearch, true, [masterKey, masterKey], done); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js index a47d6f784c..bd26091b45 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/acl/aclAwsVersioning.js @@ -86,7 +86,7 @@ function putObjectAndAcl(s3, key, body, acp, cb) { Key: key, Body: body, }); - + s3.send(command) .then(putData => { putObjectAcl(s3, key, putData.VersionId, acp, err => { @@ -113,20 +113,24 @@ function putVersionsWithAclToAws(s3, key, data, acps, cb) { throw new Error('length of data and acp arrays must be the same'); } enableVersioning(s3, bucket, () => { - async.timesLimit(data.length, 1, (i, next) => { - putObjectAndAcl(s3, key, data[i], acps[i], next); - }, (err, results) => { - if (err) { - return cb(err); - } - return cb(null, results); - }); + async.timesLimit( + data.length, + 1, + (i, next) => { + putObjectAndAcl(s3, key, data[i], acps[i], next); + }, + (err, results) => { + if (err) { + return cb(err); + } + return cb(null, results); + }, + ); }); } function getObjectAndAssertAcl(s3, params, cb) { - const { bucket, key, versionId, body, expectedVersionId, expectedResult } - = params; + const { bucket, key, versionId, body, expectedVersionId, expectedResult } = params; getAndAssertResult(s3, { bucket, key, versionId, expectedVersionId, body }) .then(() => { const aclParams = { @@ -136,13 +140,13 @@ function getObjectAndAssertAcl(s3, params, cb) { if (versionId) { aclParams.VersionId = versionId; } - + const command = new GetObjectAclCommand(aclParams); return s3.send(command); }) .then(data => { // eslint-disable-next-line no-unused-vars - const {$metadata, ...aclData} = data; + const { $metadata, ...aclData } = data; assert.deepEqual(aclData, expectedResult); cb(); }) @@ -160,23 +164,28 @@ function getObjectAndAssertAcl(s3, params, cb) { * @param {function} cb - callback * @return {undefined} - and call cb */ -function getObjectsAndAssertAcls(s3, key, versionIds, expectedData, - expectedAcps, cb) { - async.timesLimit(versionIds.length, 1, (i, next) => { - const versionId = versionIds[i]; - const body = expectedData[i]; - const expectedResult = expectedAcps[i]; - getObjectAndAssertAcl(s3, { bucket, key, versionId, body, - expectedResult, expectedVersionId: versionId }, next); - }, err => { - assert.strictEqual(err, null, 'Expected success ' + - `getting object acls, got error ${err}`); - cb(); - }); +function getObjectsAndAssertAcls(s3, key, versionIds, expectedData, expectedAcps, cb) { + async.timesLimit( + versionIds.length, + 1, + (i, next) => { + const versionId = versionIds[i]; + const body = expectedData[i]; + const expectedResult = expectedAcps[i]; + getObjectAndAssertAcl( + s3, + { bucket, key, versionId, body, expectedResult, expectedVersionId: versionId }, + next, + ); + }, + err => { + assert.strictEqual(err, null, 'Expected success ' + `getting object acls, got error ${err}`); + cb(); + }, + ); } -describeSkipIfNotMultiple('AWS backend put/get object acl with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend put/get object acl with versioning', function testSuite() { this.timeout(30000); withV4(sigCfg => { let bucketUtil; @@ -186,37 +195,35 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - + const command = new CreateBucketCommand({ Bucket: bucket, CreateBucketConfiguration: { LocationConstraint: awsLocation, }, }); - - return s3.send(command) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }); + + return s3.send(command).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('versioning not configured: should put/get acl successfully when ' + - 'versioning not configured', done => { + it('versioning not configured: should put/get acl successfully when ' + 'versioning not configured', done => { const key = `somekey-${genUniqID()}`; waitForVersioningBeforePut(s3, bucket, err => { if (err) { @@ -224,85 +231,97 @@ function testSuite() { } return putObjectAndAcl(s3, key, someBody, testAcp, (err, versionId) => { assert.strictEqual(versionId, undefined); - getObjectAndAssertAcl(s3, { bucket, key, body: someBody, - expectedResult: testAcp }, done); + getObjectAndAssertAcl(s3, { bucket, key, body: someBody, expectedResult: testAcp }, done); }); }); }); - it('versioning suspended then enabled: should put/get acl on null ' + - 'version successfully even when latest version is not null version', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - err => next(err)), - next => putVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => { - putObjectAcl(s3, key, 'null', testAcp, next); - }, - next => getObjectAndAssertAcl(s3, { bucket, key, body: '', - versionId: 'null', expectedResult: testAcp, - expectedVersionId: 'null' }, next), - ], done); - }); + it( + 'versioning suspended then enabled: should put/get acl on null ' + + 'version successfully even when latest version is not null version', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], err => next(err)), + next => putVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => { + putObjectAcl(s3, key, 'null', testAcp, next); + }, + next => + getObjectAndAssertAcl( + s3, + { + bucket, + key, + body: '', + versionId: 'null', + expectedResult: testAcp, + expectedVersionId: 'null', + }, + next, + ), + ], + done, + ); + }, + ); - it('versioning enabled: should get correct acl using version IDs', - done => { + it('versioning enabled: should get correct acl using version IDs', done => { const key = `somekey-${genUniqID()}`; - const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'] - .map(perm => { + const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'].map(perm => { const acp = new _AccessControlPolicy(ownerParams); acp.addGrantee('Group', constants.publicId, perm); return acp; }); const data = [...Array(acps.length).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => { - putObjectAndAcl(s3, key, data[0], acps[0], - () => next()); - }, - next => { - putVersionsWithAclToAws(s3, key, data.slice(1), - acps.slice(1), next); - }, - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => { - getObjectsAndAssertAcls(s3, key, versionIds, data, acps, - next); - }, - ], done); + async.waterfall( + [ + next => { + putObjectAndAcl(s3, key, data[0], acps[0], () => next()); + }, + next => { + putVersionsWithAclToAws(s3, key, data.slice(1), acps.slice(1), next); + }, + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => { + getObjectsAndAssertAcls(s3, key, versionIds, data, acps, next); + }, + ], + done, + ); }); - it('versioning enabled: should get correct acl when getting ' + - 'without version ID', done => { + it('versioning enabled: should get correct acl when getting ' + 'without version ID', done => { const key = `somekey-${genUniqID()}`; - const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'] - .map(perm => { + const acps = ['READ', 'FULL_CONTROL', 'READ_ACP', 'WRITE_ACP'].map(perm => { const acp = new _AccessControlPolicy(ownerParams); acp.addGrantee('Group', constants.publicId, perm); return acp; }); const data = [...Array(acps.length).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => putObjectAndAcl(s3, key, data[0], acps[0], - () => next()), - next => putVersionsWithAclToAws(s3, key, data.slice(1), - acps.slice(1), next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => getObjectAndAssertAcl(s3, { bucket, key, - expectedVersionId: versionIds[3], - expectedResult: acps[3], body: data[3] }, next), - ], done); + async.waterfall( + [ + next => putObjectAndAcl(s3, key, data[0], acps[0], () => next()), + next => putVersionsWithAclToAws(s3, key, data.slice(1), acps.slice(1), next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => + getObjectAndAssertAcl( + s3, + { bucket, key, expectedVersionId: versionIds[3], expectedResult: acps[3], body: data[3] }, + next, + ), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js index 138695e931..a4e107141f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/delete.js @@ -1,10 +1,5 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutObjectCommand, - DeleteObjectCommand, - GetObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); @@ -36,55 +31,67 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { process.stdout.write('Creating bucket\n'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - + await s3.send(new CreateBucketCommand({ Bucket: bucket })); - + process.stdout.write('Putting object to mem\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: memObject, - Body: body, - Metadata: { 'scal-location-constraint': memLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: memObject, + Body: body, + Metadata: { 'scal-location-constraint': memLocation }, + }), + ); + process.stdout.write('Putting object to file\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: fileObject, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: fileObject, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }), + ); + process.stdout.write('Putting object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: awsObject, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: awsObject, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); + process.stdout.write('Putting 0-byte object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: emptyObject, - Metadata: { 'scal-location-constraint': awsLocation } - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: emptyObject, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); + process.stdout.write('Putting large object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: bigObject, - Body: bigBody, - Metadata: { 'scal-location-constraint': awsLocation } - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); process.stdout.write('Putting object to AWS\n'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: mismatchObject, - Body: body, - Metadata: { 'scal-location-constraint': awsLocationMismatch } - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: mismatchObject, + Body: body, + Metadata: { 'scal-location-constraint': awsLocationMismatch }, + }), + ); }); after(async () => { @@ -106,7 +113,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete object from file', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: fileObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: fileObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -117,7 +124,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete an object from AWS', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: awsObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: awsObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -128,7 +135,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete 0-byte object from AWS', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: emptyObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: emptyObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -139,7 +146,7 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { it('should delete large object from AWS', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: bigObject })); - + try { await s3.send(new GetObjectCommand({ Bucket: bucket, Key: bigObject })); assert.fail('Expected NoSuchKey error but got success'); @@ -148,14 +155,15 @@ describeSkipIfNotMultiple('Multiple backend delete', () => { } }); - it('should delete object from AWS location with bucketMatch set to ' + - 'false', async () => { + it('should delete object from AWS location with bucketMatch set to ' + 'false', async () => { try { await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: mismatchObject })); - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: mismatchObject - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: mismatchObject, + }), + ); assert.fail('Expected NoSuchKey error but got success'); } catch (err) { assert.strictEqual(err.name, 'NoSuchKey'); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js index e8ded07b8f..fb7c7f2f08 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAwsVersioning.js @@ -7,7 +7,7 @@ const { DeleteObjectsCommand, GetObjectCommand, PutObjectCommand, - GetBucketVersioningCommand + GetBucketVersioningCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); @@ -45,22 +45,23 @@ const _deleteResultSchema = { deleteDeleteMarker: [true, true, true], }; -const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker] - = Object.keys(_deleteResultSchema); +const [nonVersionedDelete, newDeleteMarker, deleteVersion, deleteDeleteMarker] = Object.keys(_deleteResultSchema); function _assertDeleteResult(result, resultType, requestVersionId) { if (!_deleteResultSchema[resultType]) { throw new Error(`undefined result type "${resultType}"`); } - const [expectVersionId, matchReqVersionId, expectDeleteMarker] = - _deleteResultSchema[resultType]; + const [expectVersionId, matchReqVersionId, expectDeleteMarker] = _deleteResultSchema[resultType]; if (expectVersionId && matchReqVersionId) { assert.strictEqual(result.VersionId, requestVersionId); } else if (expectVersionId) { assert(result.VersionId, 'expected version id in result'); } else { - assert.strictEqual(result.VersionId, undefined, - `did not expect version id in result, got "${result.VersionId}"`); + assert.strictEqual( + result.VersionId, + undefined, + `did not expect version id in result, got "${result.VersionId}"`, + ); } if (expectDeleteMarker) { assert.strictEqual(result.DeleteMarker, true); @@ -71,24 +72,29 @@ function _assertDeleteResult(result, resultType, requestVersionId) { function delAndAssertResult(s3, params, cb) { const { bucket, key, versionId, resultType, resultError } = params; - return s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })).then(result => { - if (resultError) { - assert.fail(`expected ${resultError} but got success`); - } - _assertDeleteResult(result, resultType, versionId); - return cb(null, result.VersionId); - }).catch(err => { - if (resultError) { - assert.strictEqual(err.name, resultError); - assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); - return cb(null); - } - return cb(err); - }); + return s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ) + .then(result => { + if (resultError) { + assert.fail(`expected ${resultError} but got success`); + } + _assertDeleteResult(result, resultType, versionId); + return cb(null, result.VersionId); + }) + .catch(err => { + if (resultError) { + assert.strictEqual(err.name, resultError); + assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); + return cb(null); + } + return cb(err); + }); } function delObjectsAndAssertResult(s3, params, cb) { @@ -102,53 +108,67 @@ function delObjectsAndAssertResult(s3, params, cb) { ], Quiet: false, }; - return s3.send(new DeleteObjectsCommand({ - Bucket: bucket, - Delete: deleteParams - })).then(res => { - if (resultError) { - assert.fail(`expected ${resultError} but got success`); - } - const result = res.Deleted[0]; - _assertDeleteResult(result, resultType, versionId); - return cb(null, result.VersionId); - }).catch(err => { - if (resultError) { - assert.strictEqual(err.name, resultError); - assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); - return cb(null); - } - return cb(err); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: deleteParams, + }), + ) + .then(res => { + if (resultError) { + assert.fail(`expected ${resultError} but got success`); + } + const result = res.Deleted[0]; + _assertDeleteResult(result, resultType, versionId); + return cb(null, result.VersionId); + }) + .catch(err => { + if (resultError) { + assert.strictEqual(err.name, resultError); + assert.strictEqual(err.$metadata.httpStatusCode, errors[resultError].code); + return cb(null); + } + return cb(err); + }); } function _createDeleteMarkers(s3, bucket, key, count, cb) { - return async.timesSeries(count, - (i, next) => delAndAssertResult(s3, { bucket, key, - resultType: newDeleteMarker }, next), - cb); + return async.timesSeries( + count, + (i, next) => delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, next), + cb, + ); } function _deleteDeleteMarkers(s3, bucket, key, deleteMarkerVids, cb) { - return async.mapSeries(deleteMarkerVids, (versionId, next) => { - delAndAssertResult(s3, { bucket, key, versionId, - resultType: deleteDeleteMarker }, next); - }, () => cb()); + return async.mapSeries( + deleteMarkerVids, + (versionId, next) => { + delAndAssertResult(s3, { bucket, key, versionId, resultType: deleteDeleteMarker }, next); + }, + () => cb(), + ); } function _getAssertDeleted(s3, params, cb) { const { key, versionId, errorCode } = params; - return s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })).then(() => { - assert.fail('Expected error but got success'); - }).catch(err => { - assert.strictEqual(err.name, errorCode); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - return cb(); - }); + return s3 + .send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ) + .then(() => { + assert.fail('Expected error but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, errorCode); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + return cb(); + }); } // Update AWS S3 direct calls @@ -161,562 +181,702 @@ function _awsGetAssertDeleted(params, cb) { }); } -describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + - 'using object location constraint', function testSuite() { - this.timeout(120000); - withV4(sigCfg => { - let bucketUtil; - let s3; - beforeEach(() => { - process.stdout.write('Creating bucket\n'); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new GetBucketVersioningCommand({ Bucket: bucket }))) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; +describeSkipIfNotMultiple( + 'AWS backend delete object w. versioning: ' + 'using object location constraint', + function testSuite() { + this.timeout(120000); + withV4(sigCfg => { + let bucketUtil; + let s3; + beforeEach(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => s3.send(new GetBucketVersioningCommand({ Bucket: bucket }))) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); - }); - afterEach(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - }); - it('versioning not configured: if specifying "null" version, should ' + - 'delete specific version in AWS backend', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + it( + 'versioning not configured: if specifying "null" version, should ' + + 'delete specific version in AWS backend', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: 'null', resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); }, - ], done); - }); - - it('versioning not configured: specifying any version id other ' + - 'than null should not result in its deletion in AWS backend', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: 'awsVerIdWhichIsLongerThan40BytesButNotLongEnough', - resultError: 'InvalidArgument' }, err => next(err, awsVerId)), - (awsVerId, next) => awsGetLatestVerId(key, someBody, - (err, resultVid) => { - assert.strictEqual(resultVid, awsVerId); - next(); - }), - ], done); - }); - - it('versioning suspended: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + ); + + it( + 'versioning not configured: specifying any version id other ' + + 'than null should not result in its deletion in AWS backend', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delAndAssertResult( + s3, + { + bucket, + key, + versionId: 'awsVerIdWhichIsLongerThan40BytesButNotLongEnough', + resultError: 'InvalidArgument', + }, + err => next(err, awsVerId), + ), + (awsVerId, next) => + awsGetLatestVerId(key, someBody, (err, resultVid) => { + assert.strictEqual(resultVid, awsVerId); + next(); + }), + ], + done, + ); }, - ], done); - }); - - it('versioning enabled: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - (s3VerId, awsVerId, next) => delAndAssertResult(s3, { bucket, - key, versionId: s3VerId, resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); - }, - ], done); - }); - - it('versioning not configured: deleting existing object should ' + - 'not return version id or x-amz-delete-marker: true but should ' + - 'create a delete marker in aws ', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => delAndAssertResult(s3, { bucket, key, - resultType: nonVersionedDelete }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); + ); + + it('versioning suspended: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delAndAssertResult(s3, { bucket, key, versionId: 'null', resultType: deleteVersion }, err => + next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); - it('versioning suspended: should create a delete marker in s3 ' + - 'and aws successfully when deleting existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); + it('versioning enabled: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + (s3VerId, awsVerId, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: s3VerId, resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); - // NOTE: Normal deletes when versioning is suspended create a - // delete marker with the version id "null", which overwrites an - // existing null version in s3 metadata. - it('versioning suspended: creating a delete marker will overwrite an ' + - 'existing null version that is the latest version in s3 metadata,' + - ' but the data of the first null version will remain in AWS', - function itF(done) { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsNullVid, next) => { - this.test.awsNullVid = awsNullVid; - next(); + it( + 'versioning not configured: deleting existing object should ' + + 'not return version id or x-amz-delete-marker: true but should ' + + 'create a delete marker in aws ', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => + delAndAssertResult(s3, { bucket, key, resultType: nonVersionedDelete }, err => + next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); }, - // following call should generate a delete marker - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, next), - // delete delete marker - (dmVid, next) => delAndAssertResult(s3, { bucket, key, - versionId: dmVid, resultType: deleteDeleteMarker }, - err => next(err)), - // should get no such object even after deleting del marker - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - // get directly to aws however will give us first null version - next => awsGetLatestVerId(key, someBody, next), - (awsLatestVid, next) => { - assert.strictEqual(awsLatestVid, this.test.awsNullVid); - next(); + ); + + it( + 'versioning suspended: should create a delete marker in s3 ' + + 'and aws successfully when deleting existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); }, - ], done); - }); - - // NOTE: Normal deletes when versioning is suspended create a - // delete marker with the version id "null" which is supposed to - // overwrite any existing null version. - it('versioning suspended: creating a delete marker will overwrite an ' + - 'existing null version that is not the latest version in s3 metadata,' + - ' but the data of the first null version will remain in AWS', - function itF(done) { - const key = `somekey-${genUniqID()}`; - const data = [undefined, 'data1']; - async.waterfall([ - // put null version - next => putToAwsBackend(s3, bucket, key, data[0], - err => next(err)), - next => awsGetLatestVerId(key, '', next), - (awsNullVid, next) => { - this.test.awsNullVid = awsNullVid; - next(); + ); + + // NOTE: Normal deletes when versioning is suspended create a + // delete marker with the version id "null", which overwrites an + // existing null version in s3 metadata. + it( + 'versioning suspended: creating a delete marker will overwrite an ' + + 'existing null version that is the latest version in s3 metadata,' + + ' but the data of the first null version will remain in AWS', + function itF(done) { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsNullVid, next) => { + this.test.awsNullVid = awsNullVid; + next(); + }, + // following call should generate a delete marker + next => delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, next), + // delete delete marker + (dmVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: dmVid, resultType: deleteDeleteMarker }, + err => next(err), + ), + // should get no such object even after deleting del marker + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + // get directly to aws however will give us first null version + next => awsGetLatestVerId(key, someBody, next), + (awsLatestVid, next) => { + assert.strictEqual(awsLatestVid, this.test.awsNullVid); + next(); + }, + ], + done, + ); }, - // enable versioning and put another version - next => putVersionsToAws(s3, bucket, key, [data[1]], next), - (versions, next) => { - this.test.s3vid = versions[0]; - next(); + ); + + // NOTE: Normal deletes when versioning is suspended create a + // delete marker with the version id "null" which is supposed to + // overwrite any existing null version. + it( + 'versioning suspended: creating a delete marker will overwrite an ' + + 'existing null version that is not the latest version in s3 metadata,' + + ' but the data of the first null version will remain in AWS', + function itF(done) { + const key = `somekey-${genUniqID()}`; + const data = [undefined, 'data1']; + async.waterfall( + [ + // put null version + next => putToAwsBackend(s3, bucket, key, data[0], err => next(err)), + next => awsGetLatestVerId(key, '', next), + (awsNullVid, next) => { + this.test.awsNullVid = awsNullVid; + next(); + }, + // enable versioning and put another version + next => putVersionsToAws(s3, bucket, key, [data[1]], next), + (versions, next) => { + this.test.s3vid = versions[0]; + next(); + }, + next => suspendVersioning(s3, bucket, next), + // overwrites null version in s3 metadata but does not send + // additional delete to AWS to clean up previous "null" version + next => delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, next), + (s3dmVid, next) => { + this.test.s3DeleteMarkerId = s3dmVid; + next(); + }, + // delete delete marker + next => + delAndAssertResult( + s3, + { + bucket, + key, + versionId: this.test.s3DeleteMarkerId, + resultType: deleteDeleteMarker, + }, + err => next(err), + ), + // deleting latest version after del marker + next => + delAndAssertResult( + s3, + { bucket, key, versionId: this.test.s3vid, resultType: deleteVersion }, + err => next(err), + ), + // should get no such object instead of null version + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + // we get the null version that should have been "overwritten" + // when getting the latest version in AWS now + next => awsGetLatestVerId(key, '', next), + (awsLatestVid, next) => { + assert.strictEqual(awsLatestVid, this.test.awsNullVid); + next(); + }, + ], + done, + ); }, - next => suspendVersioning(s3, bucket, next), - // overwrites null version in s3 metadata but does not send - // additional delete to AWS to clean up previous "null" version - next => delAndAssertResult(s3, { bucket, key, - resultType: newDeleteMarker }, next), - (s3dmVid, next) => { - this.test.s3DeleteMarkerId = s3dmVid; - next(); + ); + + it( + 'versioning enabled: should create a delete marker in s3 and ' + + 'aws successfully when deleting existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); }, - // delete delete marker - next => delAndAssertResult(s3, { bucket, key, - versionId: this.test.s3DeleteMarkerId, - resultType: deleteDeleteMarker }, err => next(err)), - // deleting latest version after del marker - next => delAndAssertResult(s3, { bucket, key, - versionId: this.test.s3vid, resultType: deleteVersion }, - err => next(err)), - // should get no such object instead of null version - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - // we get the null version that should have been "overwritten" - // when getting the latest version in AWS now - next => awsGetLatestVerId(key, '', next), - (awsLatestVid, next) => { - assert.strictEqual(awsLatestVid, this.test.awsNullVid); - next(); - }, - ], done); - }); - - it('versioning enabled: should create a delete marker in s3 and ' + - 'aws successfully when deleting existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('versioning enabled: should delete a delete marker in s3 and ' + - 'aws successfully', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - // create a delete marker - (s3vid, next) => delAndAssertResult(s3, { bucket, key, - resultType: newDeleteMarker }, (err, delMarkerVid) => - next(err, s3vid, delMarkerVid)), - // delete delete marker - (s3vid, dmVid, next) => delAndAssertResult(s3, { bucket, key, - versionId: dmVid, resultType: deleteDeleteMarker }, - err => next(err, s3vid)), - // should be able to get object originally put from s3 - (s3vid, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: s3vid }, next), - // latest version in aws should now be object originally put - next => awsGetLatestVerId(key, someBody, next), - ], done); - }); - - it('multiple delete markers: should be able to get pre-existing ' + - 'versions after creating and deleting several delete markers', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => _createDeleteMarkers(s3, bucket, key, 3, - (err, dmVids) => next(err, s3vid, dmVids)), - (s3vid, dmVids, next) => _deleteDeleteMarkers(s3, bucket, key, - dmVids, () => next(null, s3vid)), - // should be able to get object originally put from s3 - (s3vid, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: s3vid }, next), - // latest version in aws should now be object originally put - next => awsGetLatestVerId(key, someBody, next), - ], done); - }); - - it('multiple delete markers: should get NoSuchObject if only ' + - 'one of the delete markers is deleted', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => _createDeleteMarkers(s3, bucket, key, 3, - (err, dmVids) => next(err, dmVids[2])), - (lastDmVid, next) => delAndAssertResult(s3, { bucket, - key, versionId: lastDmVid, resultType: deleteDeleteMarker }, - err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('should get the new latest version after deleting the latest' + - 'specific version', done => { - const key = `somekey-${genUniqID()}`; - const data = [...Array(4).keys()].map(i => i.toString()); - async.waterfall([ - // put 3 null versions - next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3), - err => next(err)), - // put one version - next => putVersionsToAws(s3, bucket, key, [data[3]], - (err, versionIds) => next(err, versionIds[0])), - // delete the latest version - (versionId, next) => delAndAssertResult(s3, { bucket, - key, versionId, resultType: deleteVersion }, - err => next(err)), - // should get the last null version - next => getAndAssertResult(s3, { bucket, key, - body: data[2], expectedVersionId: 'null' }, next), - next => awsGetLatestVerId(key, data[2], - err => next(err)), - // delete the null version - next => delAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err)), - // s3 metadata should report no existing versions for keyname - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - // NOTE: latest version in aws will be the second null version - next => awsGetLatestVerId(key, data[1], - err => next(err)), - ], done); - }); + ); + + it('versioning enabled: should delete a delete marker in s3 and ' + 'aws successfully', done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + // create a delete marker + (s3vid, next) => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, (err, delMarkerVid) => + next(err, s3vid, delMarkerVid), + ), + // delete delete marker + (s3vid, dmVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: dmVid, resultType: deleteDeleteMarker }, + err => next(err, s3vid), + ), + // should be able to get object originally put from s3 + (s3vid, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: s3vid }, next), + // latest version in aws should now be object originally put + next => awsGetLatestVerId(key, someBody, next), + ], + done, + ); + }); - it('should delete the correct version even if other versions or ' + - 'delete markers put directly on aws', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - // put an object in AWS - (s3vid, awsVid, next) => awsS3.send(new PutObjectCommand({ - Bucket: awsBucket, - Key: key - })).then(() => next(null, s3vid, awsVid)) - .catch(err => next(err)), - // create a delete marker in AWS - (s3vid, awsVid, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key - })).then(() => next(null, s3vid, awsVid)) - .catch(err => next(err)), - // delete original version in s3 - (s3vid, awsVid, next) => delAndAssertResult(s3, { bucket, key, - versionId: s3vid, resultType: deleteVersion }, - err => next(err, awsVid)), - (awsVid, next) => _getAssertDeleted(s3, { key, - errorCode: 'NoSuchKey' }, () => next(null, awsVid)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + it( + 'multiple delete markers: should be able to get pre-existing ' + + 'versions after creating and deleting several delete markers', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => + _createDeleteMarkers(s3, bucket, key, 3, (err, dmVids) => next(err, s3vid, dmVids)), + (s3vid, dmVids, next) => + _deleteDeleteMarkers(s3, bucket, key, dmVids, () => next(null, s3vid)), + // should be able to get object originally put from s3 + (s3vid, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: s3vid }, next), + // latest version in aws should now be object originally put + next => awsGetLatestVerId(key, someBody, next), + ], + done, + ); }, - ], done); - }); + ); + + it( + 'multiple delete markers: should get NoSuchObject if only ' + 'one of the delete markers is deleted', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => _createDeleteMarkers(s3, bucket, key, 3, (err, dmVids) => next(err, dmVids[2])), + (lastDmVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: lastDmVid, resultType: deleteDeleteMarker }, + err => next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); + + it('should get the new latest version after deleting the latest' + 'specific version', done => { + const key = `somekey-${genUniqID()}`; + const data = [...Array(4).keys()].map(i => i.toString()); + async.waterfall( + [ + // put 3 null versions + next => mapToAwsPuts(s3, bucket, key, data.slice(0, 3), err => next(err)), + // put one version + next => + putVersionsToAws(s3, bucket, key, [data[3]], (err, versionIds) => next(err, versionIds[0])), + // delete the latest version + (versionId, next) => + delAndAssertResult(s3, { bucket, key, versionId, resultType: deleteVersion }, err => + next(err), + ), + // should get the last null version + next => getAndAssertResult(s3, { bucket, key, body: data[2], expectedVersionId: 'null' }, next), + next => awsGetLatestVerId(key, data[2], err => next(err)), + // delete the null version + next => + delAndAssertResult(s3, { bucket, key, versionId: 'null', resultType: deleteVersion }, err => + next(err), + ), + // s3 metadata should report no existing versions for keyname + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + // NOTE: latest version in aws will be the second null version + next => awsGetLatestVerId(key, data[1], err => next(err)), + ], + done, + ); + }); - it('should not return an error deleting a version that was already ' + - 'deleted directly from AWS backend', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - // delete the object in AWS - (s3vid, awsVid, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - })).then(() => next(null, s3vid)) - .catch(err => next(err)), - // then try to delete in S3 - (s3vid, next) => delAndAssertResult(s3, { bucket, key, - versionId: s3vid, resultType: deleteVersion }, - err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - ], done); + it( + 'should delete the correct version even if other versions or ' + 'delete markers put directly on aws', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => + awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + // put an object in AWS + (s3vid, awsVid, next) => + awsS3 + .send( + new PutObjectCommand({ + Bucket: awsBucket, + Key: key, + }), + ) + .then(() => next(null, s3vid, awsVid)) + .catch(err => next(err)), + // create a delete marker in AWS + (s3vid, awsVid, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + }), + ) + .then(() => next(null, s3vid, awsVid)) + .catch(err => next(err)), + // delete original version in s3 + (s3vid, awsVid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: s3vid, resultType: deleteVersion }, + err => next(err, awsVid), + ), + (awsVid, next) => + _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, () => next(null, awsVid)), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }, + ); + + it( + 'should not return an error deleting a version that was already ' + 'deleted directly from AWS backend', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => + awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + // delete the object in AWS + (s3vid, awsVid, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }), + ) + .then(() => next(null, s3vid)) + .catch(err => next(err)), + // then try to delete in S3 + (s3vid, next) => + delAndAssertResult( + s3, + { bucket, key, versionId: s3vid, resultType: deleteVersion }, + err => next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); }); - }); -}); - -describeSkipIfNotMultiple('AWS backend delete object w. versioning: ' + - 'using bucket location constraint', function testSuite() { - this.timeout(120000); - const createBucketParams = { - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - }; - withV4(sigCfg => { - let bucketUtil; - let s3; - beforeEach(() => { - process.stdout.write('Creating bucket\n'); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand(createBucketParams)) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; + }, +); + +describeSkipIfNotMultiple( + 'AWS backend delete object w. versioning: ' + 'using bucket location constraint', + function testSuite() { + this.timeout(120000); + const createBucketParams = { + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }; + withV4(sigCfg => { + let bucketUtil; + let s3; + beforeEach(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.send(new CreateBucketCommand(createBucketParams)).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); - }); - afterEach(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - }); - it('versioning not configured: deleting non-existing object should ' + - 'not return version id or x-amz-delete-marker: true nor create a ' + - 'delete marker in aws ', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => delAndAssertResult(s3, { bucket, key, - resultType: nonVersionedDelete }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('versioning suspended: should create a delete marker in s3 ' + - 'and aws successfully when deleting non-existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => suspendVersioning(s3, bucket, next), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); - }); - - it('versioning enabled: should create a delete marker in s3 and ' + - 'aws successfully when deleting non-existing object', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => delAndAssertResult(s3, { bucket, key, resultType: - newDeleteMarker }, err => next(err)), - next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, - next), - next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, - next), - ], done); + it( + 'versioning not configured: deleting non-existing object should ' + + 'not return version id or x-amz-delete-marker: true nor create a ' + + 'delete marker in aws ', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + delAndAssertResult(s3, { bucket, key, resultType: nonVersionedDelete }, err => + next(err), + ), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); + + it( + 'versioning suspended: should create a delete marker in s3 ' + + 'and aws successfully when deleting non-existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => suspendVersioning(s3, bucket, next), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); + + it( + 'versioning enabled: should create a delete marker in s3 and ' + + 'aws successfully when deleting non-existing object', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + delAndAssertResult(s3, { bucket, key, resultType: newDeleteMarker }, err => next(err)), + next => _getAssertDeleted(s3, { key, errorCode: 'NoSuchKey' }, next), + next => _awsGetAssertDeleted({ key, errorCode: 'NoSuchKey' }, next), + ], + done, + ); + }, + ); }); - }); -}); - - -describeSkipIfNotMultiple('AWS backend delete multiple objects w. versioning: ' + - 'using object location constraint', function testSuite() { - this.timeout(120000); - withV4(sigCfg => { - let bucketUtil; - let s3; - beforeEach(() => { - process.stdout.write('Creating bucket\n'); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; + }, +); + +describeSkipIfNotMultiple( + 'AWS backend delete multiple objects w. versioning: ' + 'using object location constraint', + function testSuite() { + this.timeout(120000); + withV4(sigCfg => { + let bucketUtil; + let s3; + beforeEach(() => { + process.stdout.write('Creating bucket\n'); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); - }); - afterEach(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; + afterEach(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - }); - it('versioning not configured: if specifying "null" version, should ' + - 'delete specific version in AWS backend', done => { - const key = `somekey-${Date.now()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, someBody, - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delObjectsAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + it( + 'versioning not configured: if specifying "null" version, should ' + + 'delete specific version in AWS backend', + done => { + const key = `somekey-${Date.now()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, someBody, err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delObjectsAndAssertResult( + s3, + { bucket, key, versionId: 'null', resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); }, - ], done); - }); - - it('versioning suspended: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${Date.now()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [someBody], - err => next(err)), - next => awsGetLatestVerId(key, someBody, next), - (awsVerId, next) => delObjectsAndAssertResult(s3, { bucket, - key, versionId: 'null', resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); - }, - ], done); - }); + ); + + it('versioning suspended: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${Date.now()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [someBody], err => next(err)), + next => awsGetLatestVerId(key, someBody, next), + (awsVerId, next) => + delObjectsAndAssertResult( + s3, + { bucket, key, versionId: 'null', resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); - it('versioning enabled: should delete a specific version in AWS ' + - 'backend successfully', done => { - const key = `somekey-${Date.now()}`; - async.waterfall([ - next => waitForVersioningBeforePut(s3, bucket, next), - next => putVersionsToAws(s3, bucket, key, [someBody], - (err, versionIds) => next(err, versionIds[0])), - (s3vid, next) => awsGetLatestVerId(key, someBody, - (err, awsVid) => next(err, s3vid, awsVid)), - (s3VerId, awsVerId, next) => delObjectsAndAssertResult(s3, { bucket, - key, versionId: s3VerId, resultType: deleteVersion }, - err => next(err, awsVerId)), - (awsVerId, next) => { - _awsGetAssertDeleted({ key, - versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); - }, - ], done); + it('versioning enabled: should delete a specific version in AWS ' + 'backend successfully', done => { + const key = `somekey-${Date.now()}`; + async.waterfall( + [ + next => waitForVersioningBeforePut(s3, bucket, next), + next => + putVersionsToAws(s3, bucket, key, [someBody], (err, versionIds) => + next(err, versionIds[0]), + ), + (s3vid, next) => awsGetLatestVerId(key, someBody, (err, awsVid) => next(err, s3vid, awsVid)), + (s3VerId, awsVerId, next) => + delObjectsAndAssertResult( + s3, + { bucket, key, versionId: s3VerId, resultType: deleteVersion }, + err => next(err, awsVerId), + ), + (awsVerId, next) => { + _awsGetAssertDeleted({ key, versionId: awsVerId, errorCode: 'NoSuchVersion' }, next); + }, + ], + done, + ); + }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js index 780f8fd632..eb17789564 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteAzure.js @@ -1,11 +1,13 @@ const assert = require('assert'); const async = require('async'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, GetObjectCommand, CreateMultipartUploadCommand, - AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); + AbortMultipartUploadCommand, +} = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const withV4 = require('../../support/withV4'); const { @@ -26,12 +28,11 @@ const azureClient = getAzureClient(); const normalBody = Buffer.from('I am a body', 'utf8'); const azureTimeout = 20000; -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; -describeSkipIfNotMultiple('Multiple backend delete object from Azure', -function testSuite() { +describeSkipIfNotMultiple('Multiple backend delete object from Azure', function testSuite() { this.timeout(250000); withV4(sigCfg => { let bucketUtil; @@ -41,8 +42,7 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -50,171 +50,223 @@ function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); keys.forEach(key => { const keyName = uniqName(keyObject); describe(`${key.describe} size`, () => { before(done => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: keyName, - Body: key.body, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: keyName, + Body: key.body, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ).then(() => done()); }); - it(`should delete an ${key.describe} object from Azure`, - done => { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: keyName, - })).then(() => { - setTimeout(() => azureClient.getContainerClient(azureContainerName) - .getProperties(keyName) - .then(() => assert.fail('Expected error'), err => { - assert.strictEqual(err.statusCode, 404); - assert.strictEqual(err.code, 'NotFound'); - return done(); - }), azureTimeout); - }).catch(err => { - assert.equal(err, null, 'Expected success ' + - `but got error ${err}`); - }); + it(`should delete an ${key.describe} object from Azure`, done => { + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: keyName, + }), + ) + .then(() => { + setTimeout( + () => + azureClient + .getContainerClient(azureContainerName) + .getProperties(keyName) + .then( + () => assert.fail('Expected error'), + err => { + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NotFound'); + return done(); + }, + ), + azureTimeout, + ); + }) + .catch(err => { + assert.equal(err, null, 'Expected success ' + `but got error ${err}`); + }); }); }); }); - describe('delete from Azure location with bucketMatch set to false', - () => { + describe('delete from Azure location with bucketMatch set to false', () => { beforeEach(function beforeF(done) { this.currentTest.azureObject = uniqName(keyObject); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocationMismatch, - }, - })).then(() => done()); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocationMismatch, + }, + }), + ).then(() => done()); }); it('should delete object', function itF(done) { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.azureObject, - })).then(() => { - setTimeout(() => - azureClient.getContainerClient(azureContainerName) - .getProperties(`${azureContainerName}/${this.test.azureObject}`) - .then(() => assert.fail('Expected error'), err => { - assert.strictEqual(err.statusCode, 404); - assert.strictEqual(err.code, 'NotFound'); - return done(); - }), azureTimeout); - }).catch(err => { - assert.equal(err, null, 'Expected success ' + - `but got error ${err}`); - }); + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + }), + ) + .then(() => { + setTimeout( + () => + azureClient + .getContainerClient(azureContainerName) + .getProperties(`${azureContainerName}/${this.test.azureObject}`) + .then( + () => assert.fail('Expected error'), + err => { + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NotFound'); + return done(); + }, + ), + azureTimeout, + ); + }) + .catch(err => { + assert.equal(err, null, 'Expected success ' + `but got error ${err}`); + }); }); }); describe('returning no error', () => { beforeEach(function beF(done) { this.currentTest.azureObject = uniqName(keyObject); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => { - azureClient.getContainerClient(azureContainerName) - .deleteBlob(this.currentTest.azureObject).then(done, err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => { + azureClient + .getContainerClient(azureContainerName) + .deleteBlob(this.currentTest.azureObject) + .then(done, err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(); + }); }); - it('should return no error on deleting an object deleted ' + - 'from Azure', function itF(done) { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.azureObject, - })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(); - }); + it('should return no error on deleting an object deleted ' + 'from Azure', function itF(done) { + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(); + }); }); }); describe('Versioning:: ', () => { beforeEach(function beF(done) { this.currentTest.azureObject = uniqName(keyObject); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()); - }); - - it('should not delete object when deleting a non-existing ' + - 'version from Azure', function itF(done) { - async.waterfall([ - next => s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.azureObject, - VersionId: nonExistingId, - })).then(() => next()) - .catch(err => { - next(err); - }), - next => s3.send(new GetObjectCommand({ + s3.send( + new PutObjectCommand({ Bucket: azureContainerName, - Key: this.test.azureObject, - })).then(res => { - assert.deepStrictEqual(res.Body, normalBody); - return next(); - }).catch(err => { - assert.equal(err, null, 'getObject: Expected success ' + - `but got error ${err}`); - next(err); - }), - next => azureClient.getContainerClient(azureContainerName) - .getBlobClient(this.test.azureObject) - .downloadToBuffer().then(res => { - assert.deepStrictEqual(res, normalBody); - return next(); - }, err => { - assert.equal(err, null, 'getBlobToText: Expected ' + - `successbut got error ${err}`); - return next(); + Key: this.currentTest.azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocation, + }, }), - ], done); + ).then(() => done()); + }); + + it('should not delete object when deleting a non-existing ' + 'version from Azure', function itF(done) { + async.waterfall( + [ + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + VersionId: nonExistingId, + }), + ) + .then(() => next()) + .catch(err => { + next(err); + }), + next => + s3 + .send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: this.test.azureObject, + }), + ) + .then(res => { + assert.deepStrictEqual(res.Body, normalBody); + return next(); + }) + .catch(err => { + assert.equal(err, null, 'getObject: Expected success ' + `but got error ${err}`); + next(err); + }), + next => + azureClient + .getContainerClient(azureContainerName) + .getBlobClient(this.test.azureObject) + .downloadToBuffer() + .then( + res => { + assert.deepStrictEqual(res, normalBody); + return next(); + }, + err => { + assert.equal( + err, + null, + 'getBlobToText: Expected ' + `successbut got error ${err}`, + ); + return next(); + }, + ), + ], + done, + ); }); }); @@ -227,50 +279,60 @@ function testSuite() { Body: normalBody, Metadata: { 'scal-location-constraint': azureLocation }, }; - s3.send(new PutObjectCommand(params)).then(() => { - const params = { - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CreateMultipartUploadCommand(params)).then(res => { - this.currentTest.uploadId = res.UploadId; - setTimeout(() => done(), azureTimeout); - }).catch(err => { - assert.equal(err, null, 'Err initiating MPU on ' + - `Azure: ${err}`); + s3.send(new PutObjectCommand(params)) + .then(() => { + const params = { + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + setTimeout(() => done(), azureTimeout); + }) + .catch(err => { + assert.equal(err, null, 'Err initiating MPU on ' + `Azure: ${err}`); + done(); + }); + }) + .catch(err => { + assert.equal(err, null, 'Err putting object to Azure: ' + `${err}`); done(); }); - }).catch(err => { - assert.equal(err, null, 'Err putting object to Azure: ' + - `${err}`); - done(); - }); }); afterEach(function afF(done) { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })).then(() => { - setTimeout(() => done(), azureTimeout); - }).catch(err => { - assert.equal(err, null, `Err aborting MPU: ${err}`); - done(); - }); + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => { + setTimeout(() => done(), azureTimeout); + }) + .catch(err => { + assert.equal(err, null, `Err aborting MPU: ${err}`); + done(); + }); }); it('should return InternalError', function itFn(done) { - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - })).then(() => { - done(); - }).catch(err => { - assert.strictEqual(err.code, 'MPUinProgress'); - done(); - }); + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.strictEqual(err.code, 'MPUinProgress'); + done(); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js index 6bf5daf676..ea4773acbf 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/delete/deleteGcp.js @@ -1,16 +1,8 @@ const assert = require('assert'); -const { CreateBucketCommand, - PutObjectCommand, - DeleteObjectCommand, - GetObjectCommand } = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { - gcpLocation, - gcpLocationMismatch, - genUniqID, - describeSkipIfNotMultiple, -} = require('../utils'); +const { gcpLocation, gcpLocationMismatch, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `deletegcp${genUniqID()}`; const gcpObject = `gcpObject-${genUniqID()}`; @@ -20,8 +12,7 @@ const mismatchObject = `mismatchObject-${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); -describeSkipIfNotMultiple('Multiple backend delete', -function testSuite() { +describeSkipIfNotMultiple('Multiple backend delete', function testSuite() { this.timeout(120000); withV4(sigCfg => { let bucketUtil; @@ -31,45 +22,59 @@ function testSuite() { process.stdout.write('Creating bucket\n'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }).then(() => { - process.stdout.write('Putting object to GCP\n'); - const params = { Bucket: bucket, Key: gcpObject, Body: body, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - process.stdout.write('Putting 0-byte object to GCP\n'); - const params = { Bucket: bucket, Key: emptyObject, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - process.stdout.write('Putting large object to GCP\n'); - const params = { Bucket: bucket, Key: bigObject, - Body: bigBody, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - process.stdout.write('Putting object to GCP\n'); - const params = { Bucket: bucket, Key: mismatchObject, - Body: body, Metadata: - { 'scal-location-constraint': gcpLocationMismatch } }; - return s3.send(new PutObjectCommand(params)); - }) - .catch(err => { - process.stdout.write(`Error putting objects: ${err}\n`); - throw err; - }); + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('Putting object to GCP\n'); + const params = { + Bucket: bucket, + Key: gcpObject, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to GCP\n'); + const params = { + Bucket: bucket, + Key: emptyObject, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + process.stdout.write('Putting large object to GCP\n'); + const params = { + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + process.stdout.write('Putting object to GCP\n'); + const params = { + Bucket: bucket, + Key: mismatchObject, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }; + return s3.send(new PutObjectCommand(params)); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; + }); }); after(() => { process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket) - .catch(err => { + return bucketUtil.deleteOne(bucket).catch(err => { process.stdout.write(`Error deleting bucket: ${err}\n`); throw err; }); @@ -78,42 +83,51 @@ function testSuite() { const deleteTests = [ { msg: 'should delete object from GCP', - Bucket: bucket, Key: gcpObject, + Bucket: bucket, + Key: gcpObject, }, { msg: 'should delete 0-byte object from GCP', - Bucket: bucket, Key: emptyObject, + Bucket: bucket, + Key: emptyObject, }, { msg: 'should delete large object from GCP', - Bucket: bucket, Key: bigObject, + Bucket: bucket, + Key: bigObject, }, { - msg: 'should delete object from GCP location with ' + - 'bucketMatch set to false', - Bucket: bucket, Key: mismatchObject, + msg: 'should delete object from GCP location with ' + 'bucketMatch set to false', + Bucket: bucket, + Key: mismatchObject, }, ]; deleteTests.forEach(test => { const { msg, Bucket, Key } = test; - it(msg, done => s3.send(new DeleteObjectCommand({ Bucket, Key })) - .then(() => s3.send(new GetObjectCommand({ Bucket, Key })) + it(msg, done => + s3.send(new DeleteObjectCommand({ Bucket, Key })).then(() => + s3 + .send(new GetObjectCommand({ Bucket, Key })) + .then(() => { + assert.fail('Expected error but got success'); + }) + .catch(err => { + assert.strictEqual(err.code, 'NoSuchKey', 'Expected ' + 'error but got success'); + return done(); + }), + ), + ); + }); + + it('should return success if the object does not exist', done => + s3 + .send(new DeleteObjectCommand({ Bucket: bucket, Key: 'noop' })) .then(() => { assert.fail('Expected error but got success'); - }).catch(err => { - assert.strictEqual(err.code, 'NoSuchKey', 'Expected ' + - 'error but got success'); + }) + .catch(err => { + assert.strictEqual(err, null, `Expected success, got error ${JSON.stringify(err)}`); return done(); - }))); - }); - - it('should return success if the object does not exist', - done => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'noop' })).then(() => { - assert.fail('Expected error but got success'); - }).catch(err => { - assert.strictEqual(err, null, - `Expected success, got error ${JSON.stringify(err)}`); - return done(); - })); + })); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js index 086ae9ec33..9e1607268b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/get.js @@ -43,8 +43,7 @@ describe('Multiple backend get object', function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; const command = new CreateBucketCommand({ Bucket: bucket }); - return s3.send(command) - .catch(err => { + return s3.send(command).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -52,102 +51,102 @@ describe('Multiple backend get object', function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); // aws-sdk now (v2.363.0) returns 'UriParameterError' error - it.skip('should return an error to get request without a valid ' + - 'bucket name', - done => { - const command = new GetObjectCommand({ Bucket: '', Key: 'somekey' }); - s3.send(command) - .then(() => done(new Error('Expected failure but got success'))) - .catch(err => { - assert.notEqual(err, null, - 'Expected failure but got success'); - assert.strictEqual(err.name, 'MethodNotAllowed'); - done(); - }); - }); - it('should return NoSuchKey error when no such object', - done => { - const command = new GetObjectCommand({ Bucket: bucket, Key: 'nope' }); - s3.send(command) - .then(() => done(new Error('Expected failure but got success'))) - .catch(err => { - assert.notEqual(err, null, - 'Expected failure but got success'); - assert.strictEqual(err.name, 'NoSuchKey'); - done(); - }); - }); + it.skip('should return an error to get request without a valid ' + 'bucket name', done => { + const command = new GetObjectCommand({ Bucket: '', Key: 'somekey' }); + s3.send(command) + .then(() => done(new Error('Expected failure but got success'))) + .catch(err => { + assert.notEqual(err, null, 'Expected failure but got success'); + assert.strictEqual(err.name, 'MethodNotAllowed'); + done(); + }); + }); + it('should return NoSuchKey error when no such object', done => { + const command = new GetObjectCommand({ Bucket: bucket, Key: 'nope' }); + s3.send(command) + .then(() => done(new Error('Expected failure but got success'))) + .catch(err => { + assert.notEqual(err, null, 'Expected failure but got success'); + assert.strictEqual(err.name, 'NoSuchKey'); + done(); + }); + }); - describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + - 'location with bucketMatch: true ', () => { + describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + 'location with bucketMatch: true ', () => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': awsLocation }, - })) - .then(res => { - const uploadId = res.UploadId; - const partBody = Buffer.from('helloworld', 'utf8'); - const uploadPartInput = { - Bucket: bucket, - Key: this.currentTest.key, - PartNumber: 1, - UploadId: uploadId, - Body: partBody, - ContentLength: partBody.length, - }; - const uploadPartCommand = new UploadPartCommand(uploadPartInput); - uploadPartCommand.middlewareStack.add(next => async args => { - const headers = args.request?.headers; - if (headers) { - headers['Content-Length'] = `${partBody.length}`; - headers['x-amz-decoded-content-length'] = `${partBody.length}`; - } - return next(args); - }, { step: 'build' }); - return s3.send(uploadPartCommand) - .then(partRes => { - const eTag = partRes.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + return s3 + .send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: this.currentTest.key, - MultipartUpload: { - Parts: [ - { - ETag: eTag, - PartNumber: 1, - }, - ], - }, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => { + const uploadId = res.UploadId; + const partBody = Buffer.from('helloworld', 'utf8'); + const uploadPartInput = { + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 1, UploadId: uploadId, - })); + Body: partBody, + ContentLength: partBody.length, + }; + const uploadPartCommand = new UploadPartCommand(uploadPartInput); + uploadPartCommand.middlewareStack.add( + next => async args => { + const headers = args.request?.headers; + if (headers) { + headers['Content-Length'] = `${partBody.length}`; + headers['x-amz-decoded-content-length'] = `${partBody.length}`; + } + return next(args); + }, + { step: 'build' }, + ); + return s3.send(uploadPartCommand).then(partRes => { + const eTag = partRes.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + MultipartUpload: { + Parts: [ + { + ETag: eTag, + PartNumber: 1, + }, + ], + }, + UploadId: uploadId, + }), + ); + }); + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; }); - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); }); - it('should get object from MPU on AWS ' + - 'location with bucketMatch: true ', function it(done) { + it('should get object from MPU on AWS ' + 'location with bucketMatch: true ', function it(done) { const command = new GetObjectCommand({ Bucket: bucket, Key: this.test.key, @@ -156,75 +155,78 @@ describe('Multiple backend get object', function testSuite() { .then(res => { assert.strictEqual(res.ContentLength, 10); assert.strictEqual(res.Body.toString(), 'helloworld'); - assert.deepStrictEqual(res.Metadata, - { 'scal-location-constraint': awsLocation }); + assert.deepStrictEqual(res.Metadata, { 'scal-location-constraint': awsLocation }); done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); }); - describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + - 'location with bucketMatch: false ', () => { + describeSkipIfNotMultiple('Complete MPU then get object on AWS ' + 'location with bucketMatch: false ', () => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': awsLocationMismatch }, - })) - .then(res => { - const uploadId = res.UploadId; - const partBody = Buffer.from('helloworld', 'utf8'); - const uploadPartInput = { - Bucket: bucket, - Key: this.currentTest.key, - PartNumber: 1, - UploadId: uploadId, - Body: partBody, - ContentLength: partBody.length, - }; - const uploadPartCommand = new UploadPartCommand(uploadPartInput); - uploadPartCommand.middlewareStack.add(next => async args => { - const headers = args.request?.headers; - if (headers) { - headers['content-length'] = `${partBody.length}`; - headers['x-amz-decoded-content-length'] = `${partBody.length}`; - } - return next(args); - }, { step: 'build' }); - return s3.send(uploadPartCommand) - .then(partRes => { - const eTag = partRes.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + return s3 + .send( + new CreateMultipartUploadCommand({ Bucket: bucket, Key: this.currentTest.key, - MultipartUpload: { - Parts: [ - { - ETag: eTag, - PartNumber: 1, - }, - ], - }, + Metadata: { 'scal-location-constraint': awsLocationMismatch }, + }), + ) + .then(res => { + const uploadId = res.UploadId; + const partBody = Buffer.from('helloworld', 'utf8'); + const uploadPartInput = { + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 1, UploadId: uploadId, - })); + Body: partBody, + ContentLength: partBody.length, + }; + const uploadPartCommand = new UploadPartCommand(uploadPartInput); + uploadPartCommand.middlewareStack.add( + next => async args => { + const headers = args.request?.headers; + if (headers) { + headers['content-length'] = `${partBody.length}`; + headers['x-amz-decoded-content-length'] = `${partBody.length}`; + } + return next(args); + }, + { step: 'build' }, + ); + return s3.send(uploadPartCommand).then(partRes => { + const eTag = partRes.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + MultipartUpload: { + Parts: [ + { + ETag: eTag, + PartNumber: 1, + }, + ], + }, + UploadId: uploadId, + }), + ); + }); + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; }); - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); }); - it('should get object from MPU on AWS ' + - 'location with bucketMatch: false ', function it(done) { + it('should get object from MPU on AWS ' + 'location with bucketMatch: false ', function it(done) { const command = new GetObjectCommand({ Bucket: bucket, Key: this.test.key, @@ -233,81 +235,79 @@ describe('Multiple backend get object', function testSuite() { .then(res => { assert.strictEqual(res.ContentLength, 10); assert.strictEqual(res.Body.toString(), 'helloworld'); - assert.deepStrictEqual(res.Metadata, - { 'scal-location-constraint': awsLocationMismatch }); + assert.deepStrictEqual(res.Metadata, { 'scal-location-constraint': awsLocationMismatch }); done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); }); - describeSkipIfNotMultiple('with objects in all available backends ' + - '(mem/file/AWS)', () => { + describeSkipIfNotMultiple('with objects in all available backends ' + '(mem/file/AWS)', () => { before(() => { process.stdout.write('Putting object to mem\n'); - const memCommand = new PutObjectCommand({ - Bucket: bucket, + const memCommand = new PutObjectCommand({ + Bucket: bucket, Key: memObject, Body: body, Metadata: { 'scal-location-constraint': memLocation }, }); - return s3.send(memCommand) - .then(() => { - process.stdout.write('Putting object to file\n'); - const fileCommand = new PutObjectCommand({ - Bucket: bucket, - Key: fileObject, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation }, - }); - return s3.send(fileCommand); - }) - .then(() => { - process.stdout.write('Putting object to AWS\n'); - const awsCommand = new PutObjectCommand({ - Bucket: bucket, - Key: awsObject, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation }, - }); - return s3.send(awsCommand); - }) - .then(() => { - process.stdout.write('Putting 0-byte object to mem\n'); - const emptyCommand = new PutObjectCommand({ - Bucket: bucket, - Key: emptyObject, - Metadata: { 'scal-location-constraint': memLocation }, - }); - return s3.send(emptyCommand); - }) - .then(() => { - process.stdout.write('Putting 0-byte object to AWS\n'); - const emptyAwsCommand = new PutObjectCommand({ - Bucket: bucket, - Key: emptyAwsObject, - Metadata: { 'scal-location-constraint': awsLocation }, - }); - return s3.send(emptyAwsCommand); - }) - .then(() => { - process.stdout.write('Putting large object to AWS\n'); - const bigCommand = new PutObjectCommand({ - Bucket: bucket, - Key: bigObject, - Body: bigBody, - Metadata: { 'scal-location-constraint': awsLocation }, + return s3 + .send(memCommand) + .then(() => { + process.stdout.write('Putting object to file\n'); + const fileCommand = new PutObjectCommand({ + Bucket: bucket, + Key: fileObject, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }); + return s3.send(fileCommand); + }) + .then(() => { + process.stdout.write('Putting object to AWS\n'); + const awsCommand = new PutObjectCommand({ + Bucket: bucket, + Key: awsObject, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }); + return s3.send(awsCommand); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to mem\n'); + const emptyCommand = new PutObjectCommand({ + Bucket: bucket, + Key: emptyObject, + Metadata: { 'scal-location-constraint': memLocation }, + }); + return s3.send(emptyCommand); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to AWS\n'); + const emptyAwsCommand = new PutObjectCommand({ + Bucket: bucket, + Key: emptyAwsObject, + Metadata: { 'scal-location-constraint': awsLocation }, + }); + return s3.send(emptyAwsCommand); + }) + .then(() => { + process.stdout.write('Putting large object to AWS\n'); + const bigCommand = new PutObjectCommand({ + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }); + return s3.send(bigCommand); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; }); - return s3.send(bigCommand); - }) - .catch(err => { - process.stdout.write(`Error putting objects: ${err}\n`); - throw err; - }); }); it('should get an object from mem', done => { const command = new GetObjectCommand({ Bucket: bucket, Key: memObject }); @@ -317,8 +317,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -330,8 +329,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -343,8 +341,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got error ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got error ' + `error ${err}`); done(err); }); }); @@ -356,8 +353,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -369,8 +365,7 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); @@ -382,39 +377,36 @@ describe('Multiple backend get object', function testSuite() { done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); it('should get an object using range query from AWS', done => { - const command = new GetObjectCommand({ - Bucket: bucket, + const command = new GetObjectCommand({ + Bucket: bucket, Key: bigObject, - Range: 'bytes=0-9' + Range: 'bytes=0-9', }); s3.send(command) .then(res => { assert.strictEqual(res.ContentLength, 10); - assert.strictEqual(res.ContentRange, - `bytes 0-9/${bigBodyLen}`); + assert.strictEqual(res.ContentRange, `bytes 0-9/${bigBodyLen}`); assert.strictEqual(res.ETag, `"${bigMD5}"`); done(); }) .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); done(err); }); }); }); describeSkipIfNotMultiple('with bucketMatch set to false', () => { beforeEach(done => { - const command = new PutObjectCommand({ - Bucket: bucket, - Key: mismatchObject, + const command = new PutObjectCommand({ + Bucket: bucket, + Key: mismatchObject, Body: body, - Metadata: { 'scal-location-constraint': awsLocationMismatch } + Metadata: { 'scal-location-constraint': awsLocationMismatch }, }); s3.send(command) .then(() => done()) diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js index 5f514e2bca..6424c809ec 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAwsVersioning.js @@ -1,10 +1,7 @@ const assert = require('assert'); const async = require('async'); const withV4 = require('../../support/withV4'); -const { GetObjectCommand, - PutObjectCommand, - CreateBucketCommand, - DeleteObjectCommand } = require('@aws-sdk/client-s3'); +const { GetObjectCommand, PutObjectCommand, CreateBucketCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { awsS3, @@ -23,34 +20,36 @@ const { const someBody = 'testbody'; const bucket = `getawsversioning${genUniqID()}`; -function getAndAssertVersions(s3, bucket, key, versionIds, expectedData, - cb) { - async.mapSeries(versionIds, (versionId, next) => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, - VersionId: versionId })).then(async result => { - const resultBody = await result.Body.transformToString(); - next(null, { - VersionId: result.VersionId, - Body: resultBody +function getAndAssertVersions(s3, bucket, key, versionIds, expectedData, cb) { + async.mapSeries( + versionIds, + (versionId, next) => { + s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, VersionId: versionId })) + .then(async result => { + const resultBody = await result.Body.transformToString(); + next(null, { + VersionId: result.VersionId, + Body: resultBody, + }); + }) + .catch(err => { + next(err); }); - }) - .catch(err => { - next(err); - }); - }, (err, results) => { - if (err) { - return cb(err); - } - const resultIds = results.map(result => result.VersionId); - const resultData = results.map(result => result.Body); - assert.deepStrictEqual(resultIds, versionIds); - assert.deepStrictEqual(resultData, expectedData); - return cb(); - }); + }, + (err, results) => { + if (err) { + return cb(err); + } + const resultIds = results.map(result => result.VersionId); + const resultData = results.map(result => result.Body); + assert.deepStrictEqual(resultIds, versionIds); + assert.deepStrictEqual(resultData, expectedData); + return cb(); + }, + ); } -describeSkipIfNotMultiple('AWS backend get object with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend get object with versioning', function testSuite() { this.timeout(30000); withV4(sigCfg => { let bucketUtil; @@ -60,8 +59,7 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -69,351 +67,582 @@ function testSuite() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('should not return version ids when versioning has not been ' + - 'configured via CloudServer', done => { + it('should not return version ids when versioning has not been ' + 'configured via CloudServer', done => { const key = `somekey-${genUniqID()}`; - s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(data => { - assert.strictEqual(data.VersionId, undefined); - getAndAssertResult(s3, { bucket, key, body: someBody, - expectedVersionId: false }, done); - }).catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - done(); - }); + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(data => { + assert.strictEqual(data.VersionId, undefined); + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: false }, done); + }) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + done(); + }); }); - it('should not return version ids when versioning has not been ' + - 'configured via CloudServer, even when version id specified', done => { - const key = `somekey-${genUniqID()}`; - s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(data => { - assert.strictEqual(data.VersionId, undefined); - getAndAssertResult(s3, { bucket, key, body: someBody, - versionId: 'null', expectedVersionId: false }, done); - }).catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - done(); - }); - }); + it( + 'should not return version ids when versioning has not been ' + + 'configured via CloudServer, even when version id specified', + done => { + const key = `somekey-${genUniqID()}`; + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(data => { + assert.strictEqual(data.VersionId, undefined); + getAndAssertResult( + s3, + { bucket, key, body: someBody, versionId: 'null', expectedVersionId: false }, + done, + ); + }) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + done(); + }); + }, + ); - it('should return version id for null version when versioning ' + - 'has been configured via CloudServer', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => enableVersioning(s3, bucket, next), - // get with version id specified - next => getAndAssertResult(s3, { bucket, key, body: someBody, - versionId: 'null', expectedVersionId: 'null' }, next), - // get without version id specified - next => getAndAssertResult(s3, { bucket, key, body: someBody, - expectedVersionId: 'null' }, next), - ], done); - }); + it( + 'should return version id for null version when versioning ' + 'has been configured via CloudServer', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + next => enableVersioning(s3, bucket, next), + // get with version id specified + next => + getAndAssertResult( + s3, + { bucket, key, body: someBody, versionId: 'null', expectedVersionId: 'null' }, + next, + ), + // get without version id specified + next => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: 'null' }, next), + ], + done, + ); + }, + ); - it('should overwrite the null version if putting object twice ' + - 'before versioning is configured', done => { + it('should overwrite the null version if putting object twice ' + 'before versioning is configured', done => { const key = `somekey-${genUniqID()}`; const data = ['data1', 'data2']; - async.waterfall([ - next => mapToAwsPuts(s3, bucket, key, data, err => next(err)), - // get latest version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - expectedVersionId: false }, next), - // get specific version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - versionId: 'null', expectedVersionId: false }, next), - ], done); + async.waterfall( + [ + next => mapToAwsPuts(s3, bucket, key, data, err => next(err)), + // get latest version + next => getAndAssertResult(s3, { bucket, key, body: data[1], expectedVersionId: false }, next), + // get specific version + next => + getAndAssertResult( + s3, + { bucket, key, body: data[1], versionId: 'null', expectedVersionId: false }, + next, + ), + ], + done, + ); }); - it('should overwrite existing null version if putting object ' + - 'after suspending versioning', done => { + it('should overwrite existing null version if putting object ' + 'after suspending versioning', done => { const key = `somekey-${genUniqID()}`; const data = ['data1', 'data2']; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => suspendVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[1], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - // get latest version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - expectedVersionId: 'null' }, next), - // get specific version - next => getAndAssertResult(s3, { bucket, key, body: data[1], - versionId: 'null', expectedVersionId: 'null' }, next), - ], done); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + next(err); + }), + next => suspendVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[1], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual(err, null, 'Expected success ' + `putting object, got error ${err}`); + next(err); + }), + // get latest version + next => getAndAssertResult(s3, { bucket, key, body: data[1], expectedVersionId: 'null' }, next), + // get specific version + next => + getAndAssertResult( + s3, + { bucket, key, body: data[1], versionId: 'null', expectedVersionId: 'null' }, + next, + ), + ], + done, + ); }); - it('should overwrite null version if putting object when ' + - 'versioning is suspended after versioning enabled', done => { - const key = `somekey-${genUniqID()}`; - const data = [...Array(3).keys()].map(i => `data${i}`); - let firstVersionId; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[1], - Metadata: { 'scal-location-constraint': awsLocation } })).then(result => { - assert.notEqual(result.VersionId, 'null'); - firstVersionId = result.VersionId; - next(); - }).catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - next => suspendVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[3], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - assert.strictEqual(err, null, 'Expected success ' + - `putting object, got error ${err}`); - next(err); - }), - // get latest version - next => getAndAssertResult(s3, { bucket, key, body: data[3], - expectedVersionId: 'null' }, next), - // get specific version (null) - next => getAndAssertResult(s3, { bucket, key, body: data[3], - versionId: 'null', expectedVersionId: 'null' }, next), - // assert getting first version put for good measure - next => getAndAssertResult(s3, { bucket, key, body: data[1], - versionId: firstVersionId, - expectedVersionId: firstVersionId }, next), - ], done); - }); + it( + 'should overwrite null version if putting object when ' + + 'versioning is suspended after versioning enabled', + done => { + const key = `somekey-${genUniqID()}`; + const data = [...Array(3).keys()].map(i => `data${i}`); + let firstVersionId; + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[1], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(result => { + assert.notEqual(result.VersionId, 'null'); + firstVersionId = result.VersionId; + next(); + }) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + next => suspendVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[3], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + assert.strictEqual( + err, + null, + 'Expected success ' + `putting object, got error ${err}`, + ); + next(err); + }), + // get latest version + next => getAndAssertResult(s3, { bucket, key, body: data[3], expectedVersionId: 'null' }, next), + // get specific version (null) + next => + getAndAssertResult( + s3, + { bucket, key, body: data[3], versionId: 'null', expectedVersionId: 'null' }, + next, + ), + // assert getting first version put for good measure + next => + getAndAssertResult( + s3, + { + bucket, + key, + body: data[1], + versionId: firstVersionId, + expectedVersionId: firstVersionId, + }, + next, + ), + ], + done, + ); + }, + ); - it('should get correct data from aws backend using version IDs', - done => { + it('should get correct data from aws backend using version IDs', done => { const key = `somekey-${genUniqID()}`; const data = [...Array(5).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - next(err); - }), - next => putVersionsToAws(s3, bucket, key, data.slice(1), next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => getAndAssertVersions(s3, bucket, key, versionIds, data, - next), - ], done); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + next(err); + }), + next => putVersionsToAws(s3, bucket, key, data.slice(1), next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => getAndAssertVersions(s3, bucket, key, versionIds, data, next), + ], + done, + ); }); - it('should get correct version when getting without version ID', - done => { + it('should get correct version when getting without version ID', done => { const key = `somekey-${genUniqID()}`; const data = [...Array(5).keys()].map(i => i.toString()); const versionIds = ['null']; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: data[0], - Metadata: { 'scal-location-constraint': awsLocation } })).then(() => next()) - .catch(err => { - next(err); - }), - next => putVersionsToAws(s3, bucket, key, data.slice(1), next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => getAndAssertResult(s3, { bucket, key, body: data[4], - expectedVersionId: versionIds[4] }, next), - ], done); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(err => { + next(err); + }), + next => putVersionsToAws(s3, bucket, key, data.slice(1), next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => + getAndAssertResult(s3, { bucket, key, body: data[4], expectedVersionId: versionIds[4] }, next), + ], + done, + ); }); - it('should get correct data from aws backend using version IDs ' + - 'after putting null versions, putting versions, putting more null ' + - 'versions and then putting more versions', - done => { - const key = `somekey-${genUniqID()}`; - const data = [...Array(16).keys()].map(i => i.toString()); - // put three null versions, - // 5 real versions, - // three null versions, - // 5 versions again - const firstThreeNullVersions = data.slice(0, 3); - const firstFiveVersions = data.slice(3, 8); - const secondThreeNullVersions = data.slice(8, 11); - const secondFiveVersions = data.slice(11, 16); - const versionIds = []; - const lastNullVersion = secondThreeNullVersions[2]; - const finalDataArr = firstFiveVersions.concat([lastNullVersion]) - .concat(secondFiveVersions); - async.waterfall([ - next => mapToAwsPuts(s3, bucket, key, firstThreeNullVersions, - err => next(err)), - next => putVersionsToAws(s3, bucket, key, firstFiveVersions, - next), - (ids, next) => { - versionIds.push(...ids); - next(); - }, - next => putNullVersionsToAws(s3, bucket, key, - secondThreeNullVersions, err => next(err)), - next => putVersionsToAws(s3, bucket, key, secondFiveVersions, - next), - (ids, next) => { - versionIds.push('null'); - versionIds.push(...ids); - next(); - }, - // get versions by id - next => getAndAssertVersions(s3, bucket, key, versionIds, - finalDataArr, next), - // get and assert latest version - next => getAndAssertResult(s3, { bucket, key, body: data[16], - versionId: versionIds[versionIds.length - 1], - expectedVersionId: versionIds[versionIds.length - 1] }, - next), - ], done); - }); + it( + 'should get correct data from aws backend using version IDs ' + + 'after putting null versions, putting versions, putting more null ' + + 'versions and then putting more versions', + done => { + const key = `somekey-${genUniqID()}`; + const data = [...Array(16).keys()].map(i => i.toString()); + // put three null versions, + // 5 real versions, + // three null versions, + // 5 versions again + const firstThreeNullVersions = data.slice(0, 3); + const firstFiveVersions = data.slice(3, 8); + const secondThreeNullVersions = data.slice(8, 11); + const secondFiveVersions = data.slice(11, 16); + const versionIds = []; + const lastNullVersion = secondThreeNullVersions[2]; + const finalDataArr = firstFiveVersions.concat([lastNullVersion]).concat(secondFiveVersions); + async.waterfall( + [ + next => mapToAwsPuts(s3, bucket, key, firstThreeNullVersions, err => next(err)), + next => putVersionsToAws(s3, bucket, key, firstFiveVersions, next), + (ids, next) => { + versionIds.push(...ids); + next(); + }, + next => putNullVersionsToAws(s3, bucket, key, secondThreeNullVersions, err => next(err)), + next => putVersionsToAws(s3, bucket, key, secondFiveVersions, next), + (ids, next) => { + versionIds.push('null'); + versionIds.push(...ids); + next(); + }, + // get versions by id + next => getAndAssertVersions(s3, bucket, key, versionIds, finalDataArr, next), + // get and assert latest version + next => + getAndAssertResult( + s3, + { + bucket, + key, + body: data[16], + versionId: versionIds[versionIds.length - 1], + expectedVersionId: versionIds[versionIds.length - 1], + }, + next, + ), + ], + done, + ); + }, + ); - it('should return the correct data getting versioned object ' + - 'even if object was deleted from AWS (creating a delete marker)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // create a delete marker in AWS - (versionId, next) => awsS3.deleteObject({ Bucket: awsBucket, - Key: key }, err => next(err, versionId)), - (versionId, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: versionId }, next), - ], done); - }); + it( + 'should return the correct data getting versioned object ' + + 'even if object was deleted from AWS (creating a delete marker)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // create a delete marker in AWS + (versionId, next) => + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => next(err, versionId)), + (versionId, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: versionId }, next), + ], + done, + ); + }, + ); - it('should return the correct data getting versioned object ' + - 'even if object is put directly to AWS (creating new version)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // put an object in AWS - (versionId, next) => awsS3.send(new PutObjectCommand({ Bucket: awsBucket, - Key: key })).then(() => next(null, versionId)) - .catch(err => { - next(err); - }), - (versionId, next) => getAndAssertResult(s3, { bucket, key, - body: someBody, expectedVersionId: versionId }, next), - ], done); - }); + it( + 'should return the correct data getting versioned object ' + + 'even if object is put directly to AWS (creating new version)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // put an object in AWS + (versionId, next) => + awsS3 + .send(new PutObjectCommand({ Bucket: awsBucket, Key: key })) + .then(() => next(null, versionId)) + .catch(err => { + next(err); + }), + (versionId, next) => + getAndAssertResult(s3, { bucket, key, body: someBody, expectedVersionId: versionId }, next), + ], + done, + ); + }, + ); - it('should return a LocationNotFound if trying to get an object ' + - 'that was deleted in AWS but exists in s3 metadata', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // get the latest version id in aws - (s3vid, next) => awsS3.send(new GetObjectCommand({ Bucket: awsBucket, - Key: key })).then(res => next(null, s3vid, res.VersionId)) - .catch(err => { - next(err); - }), - (s3VerId, awsVerId, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, Key: key, VersionId: awsVerId })).then(() => next(null, s3VerId)) - .catch(err => { - next(err); - }), - (s3VerId, next) => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) - .then(res => next(null, s3VerId, res.VersionId)) - .catch(err => { - assert.strictEqual(err.name, 'LocationNotFound'); - assert.strictEqual(err.$metadata.httpStatusCode, 424); - next(); - }), - ], done); - }); + it( + 'should return a LocationNotFound if trying to get an object ' + + 'that was deleted in AWS but exists in s3 metadata', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // get the latest version id in aws + (s3vid, next) => + awsS3 + .send(new GetObjectCommand({ Bucket: awsBucket, Key: key })) + .then(res => next(null, s3vid, res.VersionId)) + .catch(err => { + next(err); + }), + (s3VerId, awsVerId, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVerId, + }), + ) + .then(() => next(null, s3VerId)) + .catch(err => { + next(err); + }), + (s3VerId, next) => + s3 + .send(new GetObjectCommand({ Bucket: bucket, Key: key })) + .then(res => next(null, s3VerId, res.VersionId)) + .catch(err => { + assert.strictEqual(err.name, 'LocationNotFound'); + assert.strictEqual(err.$metadata.httpStatusCode, 424); + next(); + }), + ], + done, + ); + }, + ); - it('should return a LocationNotFound if trying to get a version ' + - 'that was deleted in AWS but exists in s3 metadata', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody, - Metadata: { 'scal-location-constraint': awsLocation } })).then(res => next(null, res.VersionId)) - .catch(err => { - next(err); - }), - // get the latest version id in aws - (s3vid, next) => awsS3.send(new GetObjectCommand({ Bucket: awsBucket, - Key: key })).then(res => next(null, s3vid, res.VersionId)) - .catch(err => { - next(err); - }), - (s3VerId, awsVerId, next) => awsS3.send(new DeleteObjectCommand({ - Bucket: awsBucket, Key: key, VersionId: awsVerId })).then(() => next(null, s3VerId)) - .catch(err => { - next(err); - }), - (s3VerId, next) => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key, - VersionId: s3VerId })).then(() => { - next(); - }).catch(err => { - assert.strictEqual(err.name, 'LocationNotFound'); - assert.strictEqual(err.$metadata.httpStatusCode, 424); - next(); - }), - ], done); - }); + it( + 'should return a LocationNotFound if trying to get a version ' + + 'that was deleted in AWS but exists in s3 metadata', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(res => next(null, res.VersionId)) + .catch(err => { + next(err); + }), + // get the latest version id in aws + (s3vid, next) => + awsS3 + .send(new GetObjectCommand({ Bucket: awsBucket, Key: key })) + .then(res => next(null, s3vid, res.VersionId)) + .catch(err => { + next(err); + }), + (s3VerId, awsVerId, next) => + awsS3 + .send( + new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVerId, + }), + ) + .then(() => next(null, s3VerId)) + .catch(err => { + next(err); + }), + (s3VerId, next) => + s3 + .send(new GetObjectCommand({ Bucket: bucket, Key: key, VersionId: s3VerId })) + .then(() => { + next(); + }) + .catch(err => { + assert.strictEqual(err.name, 'LocationNotFound'); + assert.strictEqual(err.$metadata.httpStatusCode, 424); + next(); + }), + ], + done, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js index 7f1d5e25dc..a333b47ce0 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getAzure.js @@ -1,7 +1,5 @@ const assert = require('assert'); -const { CreateBucketCommand, - PutObjectCommand, - GetObjectCommand } = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); const withV4 = require('../../support/withV4'); @@ -24,8 +22,7 @@ const normalBody = Buffer.from('I am a body', 'utf8'); const azureTimeout = 10000; -describeSkipIfNotMultiple('Multiple backend get object from Azure', -function testSuite() { +describeSkipIfNotMultiple('Multiple backend get object from Azure', function testSuite() { this.timeout(30000); withV4(sigCfg => { let bucketUtil; @@ -35,8 +32,7 @@ function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -44,46 +40,49 @@ function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); keys.forEach(key => { describe(`${key.describe} size`, () => { const testKey = `${key.name}-${Date.now()}`; before(done => { setTimeout(() => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: testKey, - Body: key.body, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()) - .catch(err => { - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: testKey, + Body: key.body, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => done()) + .catch(err => { + done(err); + }); }, azureTimeout); }); it(`should get an ${key.describe} object from Azure`, done => { - s3.send(new GetObjectCommand({ Bucket: azureContainerName, Key: - testKey })).then(res => { - assert.strictEqual(res.ETag, `"${key.MD5}"`); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success ' + - `but got error ${err}`); - done(err); - }); + s3.send(new GetObjectCommand({ Bucket: azureContainerName, Key: testKey })) + .then(res => { + assert.strictEqual(res.ETag, `"${key.MD5}"`); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success ' + `but got error ${err}`); + done(err); + }); }); }); }); @@ -91,93 +90,106 @@ function testSuite() { describe('with range', () => { const azureObject = uniqName(keyObject); before(done => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Body: '0123456789', - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => done()) - .catch(err => { - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Body: '0123456789', + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => done()) + .catch(err => { + done(err); + }); }); - it('should get an object with body 012345 with "bytes=0-5"', - done => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Range: 'bytes=0-5', - })).then(async res => { - const body = await res.Body.transformToString(); - assert.equal(res.ContentLength, 6); - assert.strictEqual(res.ContentRange, 'bytes 0-5/10'); - assert.strictEqual(body, '012345'); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); + it('should get an object with body 012345 with "bytes=0-5"', done => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Range: 'bytes=0-5', + }), + ) + .then(async res => { + const body = await res.Body.transformToString(); + assert.equal(res.ContentLength, 6); + assert.strictEqual(res.ContentRange, 'bytes 0-5/10'); + assert.strictEqual(body, '012345'); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); }); - it('should get an object with body 456789 with "bytes=4-"', - done => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Range: 'bytes=4-', - })).then(async res => { - const body = await res.Body.transformToString(); - assert.equal(res.ContentLength, 6); - assert.strictEqual(res.ContentRange, 'bytes 4-9/10'); - assert.strictEqual(body, '456789'); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); + it('should get an object with body 456789 with "bytes=4-"', done => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Range: 'bytes=4-', + }), + ) + .then(async res => { + const body = await res.Body.transformToString(); + assert.equal(res.ContentLength, 6); + assert.strictEqual(res.ContentRange, 'bytes 4-9/10'); + assert.strictEqual(body, '456789'); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); }); }); describe('returning error', () => { const azureObject = uniqName(keyObject); before(done => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - Body: normalBody, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })).then(() => { - azureClient.getContainerClient(azureContainerName) - .deleteBlob(azureObject).then(done, err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); - }) - .catch(err => { - assert.equal(err, null, 'Expected success but got ' + - `error ${err}`); - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + Body: normalBody, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(() => { + azureClient + .getContainerClient(azureContainerName) + .deleteBlob(azureObject) + .then(done, err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); + }) + .catch(err => { + assert.equal(err, null, 'Expected success but got ' + `error ${err}`); + done(err); + }); }); - it('should return an error on get done to object deleted ' + - 'from Azure', done => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: azureObject, - })).then(() => { - done(); - }).catch(err => { - assert.strictEqual(err.name, 'LocationNotFound'); - done(err); - }); + it('should return an error on get done to object deleted ' + 'from Azure', done => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: azureObject, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.strictEqual(err.name, 'LocationNotFound'); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js index a0895a7b48..acb5525a6b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/get/getGcp.js @@ -1,15 +1,8 @@ const assert = require('assert'); const withV4 = require('../../support/withV4'); -const { PutObjectCommand, - GetObjectCommand, - CreateBucketCommand } = require('@aws-sdk/client-s3'); +const { PutObjectCommand, GetObjectCommand, CreateBucketCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { - gcpLocation, - gcpLocationMismatch, - genUniqID, - describeSkipIfNotMultiple, -} = require('../utils'); +const { gcpLocation, gcpLocationMismatch, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `getgcp${genUniqID()}`; const gcpObject = `gcpobject-${genUniqID()}`; @@ -32,8 +25,7 @@ describe('Multiple backend get object', function testSuite() { process.stdout.write('Creating bucket'); bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -41,114 +33,128 @@ describe('Multiple backend get object', function testSuite() { after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); describeSkipIfNotMultiple('with objects in GCP', () => { before(() => { process.stdout.write('Putting object to GCP\n'); - return s3.send(new PutObjectCommand({ Bucket: bucket, Key: gcpObject, - Body: body, - Metadata: { 'scal-location-constraint': gcpLocation }, - }) - .then(() => { - process.stdout.write('Putting 0-byte object to GCP\n'); - return s3.send(new PutObjectCommand({ Bucket: bucket, - Key: emptyGcpObject, + return s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: gcpObject, + Body: body, Metadata: { 'scal-location-constraint': gcpLocation }, - })); - }) - .then(() => { - process.stdout.write('Putting large object to GCP\n'); - return s3.send(new PutObjectCommand({ Bucket: bucket, - Key: bigObject, Body: bigBody, - Metadata: { 'scal-location-constraint': gcpLocation }, - })); - }) - .catch(err => { - process.stdout.write(`Error putting objects: ${err}\n`); - throw err; - })); + }) + .then(() => { + process.stdout.write('Putting 0-byte object to GCP\n'); + return s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: emptyGcpObject, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ); + }) + .then(() => { + process.stdout.write('Putting large object to GCP\n'); + return s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: bigObject, + Body: bigBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ); + }) + .catch(err => { + process.stdout.write(`Error putting objects: ${err}\n`); + throw err; + }), + ); }); const getTests = [ { msg: 'should get a 0-byte object from GCP', - input: { Bucket: bucket, Key: emptyGcpObject, - range: null, size: null }, + input: { Bucket: bucket, Key: emptyGcpObject, range: null, size: null }, output: { MD5: emptyMD5, contentRange: null }, }, { msg: 'should get an object from GCP', - input: { Bucket: bucket, Key: gcpObject, - range: null, size: null }, + input: { Bucket: bucket, Key: gcpObject, range: null, size: null }, output: { MD5: correctMD5, contentRange: null }, }, { msg: 'should get a large object from GCP', - input: { Bucket: bucket, Key: bigObject, - range: null, size: null }, + input: { Bucket: bucket, Key: bigObject, range: null, size: null }, output: { MD5: bigMD5, contentRange: null }, }, { msg: 'should get an object using range query from GCP', - input: { Bucket: bucket, Key: bigObject, - range: 'bytes=0-9', size: 10 }, - output: { MD5: bigMD5, - contentRange: `bytes 0-9/${bigBodyLen}` }, + input: { Bucket: bucket, Key: bigObject, range: 'bytes=0-9', size: 10 }, + output: { MD5: bigMD5, contentRange: `bytes 0-9/${bigBodyLen}` }, }, ]; getTests.forEach(test => { const { Bucket, Key, range, size } = test.input; const { MD5, contentRange } = test.output; it(test.msg, done => { - s3.send(new GetObjectCommand({ Bucket, Key, Range: range })).then(res => { - if (range) { - assert.strictEqual(res.ContentLength, size); - assert.strictEqual(res.ContentRange, contentRange); - } - assert.strictEqual(res.ETag, `"${MD5}"`); - done(); - }) - .catch(err => { - assert.equal(err, null, - `Expected success but got error ${err}`); - done(err); - }); + s3.send(new GetObjectCommand({ Bucket, Key, Range: range })) + .then(res => { + if (range) { + assert.strictEqual(res.ContentLength, size); + assert.strictEqual(res.ContentRange, contentRange); + } + assert.strictEqual(res.ETag, `"${MD5}"`); + done(); + }) + .catch(err => { + assert.equal(err, null, `Expected success but got error ${err}`); + done(err); + }); }); }); }); describeSkipIfNotMultiple('with bucketMatch set to false', () => { beforeEach(done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: mismatchObject, Body: body, - Metadata: { 'scal-location-constraint': gcpLocationMismatch } })).then(() => { - done(); - }) - .catch(err => { - assert.equal(err, null, `Err putting object: ${err}`); - done(err); - }); + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: mismatchObject, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, `Err putting object: ${err}`); + done(err); + }); }); it('should get an object from GCP', done => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: mismatchObject })).then(res => { - assert.strictEqual(res.ETag, `"${correctMD5}"`); - done(); - }) - .catch(err => { - assert.equal(err, null, `Error getting object: ${err}`); - done(err); - }); + s3.send(new GetObjectCommand({ Bucket: bucket, Key: mismatchObject })) + .then(res => { + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); + }) + .catch(err => { + assert.equal(err, null, `Error getting object: ${err}`); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js index 33c131ff31..4b99c9fc7b 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUAzure.js @@ -9,8 +9,7 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { azureLocation, getAzureContainerName, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { azureLocation, getAzureContainerName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const keyName = `somekey-${genUniqID()}`; @@ -27,26 +26,31 @@ describeSkipIfNotMultiple('Initiate MPU to AZURE', () => { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(done => - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - CreateBucketConfiguration: { - LocationConstraint: azureLocation, - }, - })) - .then(() => done()) - .catch(done)); + s3 + .send( + new CreateBucketCommand({ + Bucket: azureContainerName, + CreateBucketConfiguration: { + LocationConstraint: azureLocation, + }, + }), + ) + .then(() => done()) + .catch(done), + ); afterEach(function afterEachF(done) { const params = { Bucket: azureContainerName, @@ -57,43 +61,43 @@ describeSkipIfNotMultiple('Initiate MPU to AZURE', () => { .then(() => done()) .catch(done); }); - it('should create MPU and list in-progress multipart uploads', - function ifF(done) { + it('should create MPU and list in-progress multipart uploads', function ifF(done) { const params = { Bucket: azureContainerName, Key: keyName, Metadata: { 'scal-location-constraint': azureLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - this.test.uploadId = res.UploadId; - assert(this.test.uploadId); - assert.strictEqual(res.Bucket, - azureContainerName); - assert.strictEqual(res.Key, keyName); - next(); - }) - .catch(next); - }, - next => { - s3.send(new ListMultipartUploadsCommand({ - Bucket: azureContainerName, - })) - .then(res => { - assert.strictEqual(res.NextKeyMarker, keyName); - assert.strictEqual(res.NextUploadIdMarker, - this.test.uploadId); - assert.strictEqual(res.Uploads[0].Key, - keyName); - assert.strictEqual(res.Uploads[0].UploadId, - this.test.uploadId); - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.test.uploadId = res.UploadId; + assert(this.test.uploadId); + assert.strictEqual(res.Bucket, azureContainerName); + assert.strictEqual(res.Key, keyName); + next(); + }) + .catch(next); + }, + next => { + s3.send( + new ListMultipartUploadsCommand({ + Bucket: azureContainerName, + }), + ) + .then(res => { + assert.strictEqual(res.NextKeyMarker, keyName); + assert.strictEqual(res.NextUploadIdMarker, this.test.uploadId); + assert.strictEqual(res.Uploads[0].Key, keyName); + assert.strictEqual(res.Uploads[0].UploadId, this.test.uploadId); + next(); + }) + .catch(next); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js index e2075dbd12..c8b7277d74 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/initMPU/initMPUGcp.js @@ -10,8 +10,7 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucketMPU, gcpLocation, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { gcpClient, gcpBucketMPU, gcpLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const { createMpuKey } = arsenal.storage.data.external.GcpUtils; const bucket = `initmpugcp${genUniqID()}`; @@ -29,26 +28,31 @@ describeSkipIfNotMultiple('Initiate MPU to GCP', () => { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(done => - s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: gcpLocation, - }, - })) - .then(() => done()) - .catch(done)); + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }), + ) + .then(() => done()) + .catch(done), + ); afterEach(function afterEachF(done) { const params = { Bucket: bucket, @@ -59,54 +63,54 @@ describeSkipIfNotMultiple('Initiate MPU to GCP', () => { .then(() => done()) .catch(done); }); - it('should create MPU and list in-progress multipart uploads', - function ifF(done) { + it('should create MPU and list in-progress multipart uploads', function ifF(done) { const params = { Bucket: bucket, Key: keyName, Metadata: { 'scal-location-constraint': gcpLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - this.test.uploadId = res.UploadId; - assert(this.test.uploadId); - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, keyName); - next(); - }) - .catch(next); - }, - next => { - s3.send(new ListMultipartUploadsCommand({ - Bucket: bucket, - })) - .then(res => { - assert.strictEqual(res.NextKeyMarker, keyName); - assert.strictEqual(res.NextUploadIdMarker, - this.test.uploadId); - assert.strictEqual(res.Uploads[0].Key, keyName); - assert.strictEqual(res.Uploads[0].UploadId, - this.test.uploadId); + async.waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.test.uploadId = res.UploadId; + assert(this.test.uploadId); + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, keyName); + next(); + }) + .catch(next); + }, + next => { + s3.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + }), + ) + .then(res => { + assert.strictEqual(res.NextKeyMarker, keyName); + assert.strictEqual(res.NextUploadIdMarker, this.test.uploadId); + assert.strictEqual(res.Uploads[0].Key, keyName); + assert.strictEqual(res.Uploads[0].UploadId, this.test.uploadId); + next(); + }) + .catch(next); + }, + next => { + const mpuKey = createMpuKey(keyName, this.test.uploadId, 'init'); + const params = { + Bucket: gcpBucketMPU, + Key: mpuKey, + }; + gcpClient.getObject(params, err => { + assert.ifError(err, `Expected success, but got err ${err}`); next(); - }) - .catch(next); - }, - next => { - const mpuKey = - createMpuKey(keyName, this.test.uploadId, 'init'); - const params = { - Bucket: gcpBucketMPU, - Key: mpuKey, - }; - gcpClient.getObject(params, err => { - assert.ifError(err, - `Expected success, but got err ${err}`); - next(); - }); - }, - ], done); + }); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js index 385b60b523..1e129fe76c 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/azureListParts.js @@ -1,14 +1,15 @@ const assert = require('assert'); -const { CreateBucketCommand, +const { + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand, - ListPartsCommand } = require('@aws-sdk/client-s3'); + ListPartsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { azureLocation, getAzureContainerName, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { azureLocation, getAzureContainerName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const azureContainerName = getAzureContainerName(azureLocation); const firstPartSize = 10; @@ -19,88 +20,121 @@ const bodySecondPart = Buffer.alloc(secondPartSize); let bucketUtil; let s3; -describeSkipIfNotMultiple('List parts of MPU on Azure data backend', -() => { +describeSkipIfNotMultiple('List parts of MPU on Azure data backend', () => { withV4(sigCfg => { beforeEach(function beforeEachFn() { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - return s3.send(new UploadPartCommand({ Bucket: azureContainerName, - Key: this.currentTest.key, PartNumber: 1, - UploadId: this.currentTest.uploadId, Body: bodyFirstPart, - })); - }).then(res => { - this.currentTest.firstEtag = res.ETag; - }).then(() => s3.send(new UploadPartCommand({ Bucket: azureContainerName, - Key: this.currentTest.key, PartNumber: 2, - UploadId: this.currentTest.uploadId, Body: bodySecondPart, - }))).then(res => { - this.currentTest.secondEtag = res.ETag; - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - })); + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).then(() => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + PartNumber: 1, + UploadId: this.currentTest.uploadId, + Body: bodyFirstPart, + }), + ); + }) + .then(res => { + this.currentTest.firstEtag = res.ETag; + }) + .then(() => + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + PartNumber: 2, + UploadId: this.currentTest.uploadId, + Body: bodySecondPart, + }), + ), + ) + .then(res => { + this.currentTest.secondEtag = res.ETag; + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; + }), + ); }); afterEach(function afterEachFn() { process.stdout.write('Emptying bucket'); - return s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => bucketUtil.empty(azureContainerName)) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => bucketUtil.empty(azureContainerName)) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); it('should list both parts', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts.length, 2); - assert.strictEqual(data.Parts[0].PartNumber, 1); - assert.strictEqual(data.Parts[0].Size, firstPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); - assert.strictEqual(data.Parts[1].PartNumber, 2); - assert.strictEqual(data.Parts[1].Size, secondPartSize); - assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts.length, 2); + assert.strictEqual(data.Parts[0].PartNumber, 1); + assert.strictEqual(data.Parts[0].Size, firstPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); + assert.strictEqual(data.Parts[1].PartNumber, 2); + assert.strictEqual(data.Parts[1].Size, secondPartSize); + assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); it('should only list the second part', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: this.test.key, - PartNumberMarker: 1, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts[0].PartNumber, 2); - assert.strictEqual(data.Parts[0].Size, secondPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: this.test.key, + PartNumberMarker: 1, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts[0].PartNumber, 2); + assert.strictEqual(data.Parts[0].Size, secondPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js index d1093d04f5..fecfa76383 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/listParts/listPartsGcp.js @@ -1,13 +1,14 @@ const assert = require('assert'); -const { CreateBucketCommand, +const { + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, AbortMultipartUploadCommand, - ListPartsCommand } = require('@aws-sdk/client-s3'); + ListPartsCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpLocation, genUniqID, describeSkipIfNotMultiple } - = require('../utils'); +const { gcpLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `listpartsgcp${genUniqID()}`; const firstPartSize = 10; @@ -24,82 +25,116 @@ describeSkipIfNotMultiple('List parts of MPU on GCP data backend', () => { this.currentTest.key = `somekey-${genUniqID()}`; bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - }))) - .then(res => { - this.currentTest.uploadId = res.UploadId; - return s3.send(new UploadPartCommand({ Bucket: bucket, - Key: this.currentTest.key, PartNumber: 1, - UploadId: this.currentTest.uploadId, Body: bodyFirstPart, - })); - }).then(res => { - this.currentTest.firstEtag = res.ETag; - }).then(() => s3.send(new UploadPartCommand({ Bucket: bucket, - Key: this.currentTest.key, PartNumber: 2, - UploadId: this.currentTest.uploadId, Body: bodySecondPart, - }))) - .then(res => { - this.currentTest.secondEtag = res.ETag; - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 1, + UploadId: this.currentTest.uploadId, + Body: bodyFirstPart, + }), + ); + }) + .then(res => { + this.currentTest.firstEtag = res.ETag; + }) + .then(() => + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: this.currentTest.key, + PartNumber: 2, + UploadId: this.currentTest.uploadId, + Body: bodySecondPart, + }), + ), + ) + .then(res => { + this.currentTest.secondEtag = res.ETag; + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; + }); }); afterEach(function afterEachFn() { process.stdout.write('Emptying bucket'); - return s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => bucketUtil.empty(bucket)) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => bucketUtil.empty(bucket)) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); it('should list both parts', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts.length, 2); - assert.strictEqual(data.Parts[0].PartNumber, 1); - assert.strictEqual(data.Parts[0].Size, firstPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); - assert.strictEqual(data.Parts[1].PartNumber, 2); - assert.strictEqual(data.Parts[1].Size, secondPartSize); - assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts.length, 2); + assert.strictEqual(data.Parts[0].PartNumber, 1); + assert.strictEqual(data.Parts[0].Size, firstPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.firstEtag); + assert.strictEqual(data.Parts[1].PartNumber, 2); + assert.strictEqual(data.Parts[1].Size, secondPartSize); + assert.strictEqual(data.Parts[1].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); it('should only list the second part', function itFn(done) { - s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: this.test.key, - PartNumberMarker: 1, - UploadId: this.test.uploadId })).then(data => { - assert.strictEqual(data.Parts[0].PartNumber, 2); - assert.strictEqual(data.Parts[0].Size, secondPartSize); - assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); - done(); - }).catch(err => { - assert.equal(err, null, `Err listing parts: ${err}`); - done(err); - }); + s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: this.test.key, + PartNumberMarker: 1, + UploadId: this.test.uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.Parts[0].PartNumber, 2); + assert.strictEqual(data.Parts[0].Size, secondPartSize); + assert.strictEqual(data.Parts[0].ETag, this.test.secondEtag); + done(); + }) + .catch(err => { + assert.equal(err, null, `Err listing parts: ${err}`); + done(err); + }); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js index 8133eafcfb..05b0f91ad1 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/abortMPUGcp.js @@ -11,8 +11,15 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucket, gcpBucketMPU, - gcpLocation, uniqName, genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { + gcpClient, + gcpBucket, + gcpBucketMPU, + gcpLocation, + uniqName, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const keyObject = 'abortgcp'; const bucket = `abortmpugcp${genUniqID()}`; @@ -30,16 +37,13 @@ function checkMPUList(bucket, key, uploadId, cb) { UploadId: uploadId, }; gcpClient.listParts(params, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.deepStrictEqual(res.Contents, [], - 'Expected 0 parts, listed some'); + assert.ifError(err, `Expected success, but got err ${err}`); + assert.deepStrictEqual(res.Contents, [], 'Expected 0 parts, listed some'); cb(); }); } -describeSkipIfNotMultiple('Abort MPU on GCP data backend', function -descrbeFn() { +describeSkipIfNotMultiple('Abort MPU on GCP data backend', function descrbeFn() { this.timeout(180000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -50,33 +54,42 @@ descrbeFn() { describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: bucket, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(done => { - s3.send(new DeleteBucketCommand({ - Bucket: bucket, - })) + s3.send( + new DeleteBucketCommand({ + Bucket: bucket, + }), + ) .then(() => done()) .catch(done); }); @@ -87,16 +100,21 @@ descrbeFn() { Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => setTimeout(() => checkMPUList( - gcpBucketMPU, this.test.key, this.test.uploadId, next), - gcpTimeout), - ], done); + async.waterfall( + [ + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => + setTimeout( + () => checkMPUList(gcpBucketMPU, this.test.key, this.test.uploadId, next), + gcpTimeout, + ), + ], + done, + ); }); it('should abort a MPU with uploaded parts', function itFn(done) { @@ -105,133 +123,149 @@ descrbeFn() { Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - async.times(2, (n, cb) => { - const uploadParams = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - Body: body, - PartNumber: n + 1, - }; - s3.send(new UploadPartCommand(uploadParams)) - .then(res => { - assert.strictEqual( - res.ETag, `"${correctMD5}"`); - cb(); - }) - .catch(err => { - assert.ifError(err, - `Expected success, but got err ${err}`); - cb(err); - }); - }, err => next(err)); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => setTimeout(() => checkMPUList( - gcpBucketMPU, this.test.key, this.test.uploadId, next), - gcpTimeout), - ], done); + async.waterfall( + [ + next => { + async.times( + 2, + (n, cb) => { + const uploadParams = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + Body: body, + PartNumber: n + 1, + }; + s3.send(new UploadPartCommand(uploadParams)) + .then(res => { + assert.strictEqual(res.ETag, `"${correctMD5}"`); + cb(); + }) + .catch(err => { + assert.ifError(err, `Expected success, but got err ${err}`); + cb(err); + }); + }, + err => next(err), + ); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => + setTimeout( + () => checkMPUList(gcpBucketMPU, this.test.key, this.test.uploadId, next), + gcpTimeout, + ), + ], + done, + ); }); }); describe('with previously existing object with same key', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: bucket, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': gcpLocation }, - Body: body, - })) - .then(() => next()) - .catch(err => { - assert.ifError(err, - `Expected success, got error: ${err}`); - next(err); - }); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': gcpLocation, + }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => { + assert.ifError(err, `Expected success, got error: ${err}`); + next(err); + }); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('should abort MPU without deleting existing object', - function itFn(done) { + it('should abort MPU without deleting existing object', function itFn(done) { const params = { Bucket: bucket, Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const body = Buffer.alloc(10); - const partParams = Object.assign( - { PartNumber: 1, Body: body }, params); - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(err => { - assert.ifError(err, - `Expected success, got error: ${err}`); - next(err); - }); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => setTimeout(() => { - const params = { - Bucket: gcpBucket, - Key: this.test.key, - }; - gcpClient.getObject(params, (err, res) => { - assert.ifError(err, - `Expected success, got error: ${err}`); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - next(); - }); - }, gcpTimeout), - ], done); + async.waterfall( + [ + next => { + const body = Buffer.alloc(10); + const partParams = Object.assign({ PartNumber: 1, Body: body }, params); + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(err => { + assert.ifError(err, `Expected success, got error: ${err}`); + next(err); + }); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => + setTimeout(() => { + const params = { + Bucket: gcpBucket, + Key: this.test.key, + }; + gcpClient.getObject(params, (err, res) => { + assert.ifError(err, `Expected success, got error: ${err}`); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + next(); + }); + }, gcpTimeout), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js index fe74499b07..741d7cba80 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuAbort/azureAbortMPU.js @@ -12,9 +12,14 @@ const { const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { uniqName, getAzureClient, - getAzureContainerName, convertMD5, azureLocation, - describeSkipIfNotMultiple } = require('../utils'); +const { + uniqName, + getAzureClient, + getAzureContainerName, + convertMD5, + azureLocation, + describeSkipIfNotMultiple, +} = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const maxSubPartSize = azureMpuUtils.maxSubPartSize; @@ -27,22 +32,26 @@ let bucketUtil; let s3; function azureCheck(container, key, expected, cb) { - azureClient.getContainerClient(container).getProperties(key).then(res => { - assert.ok(!expected.error); - const convertedMD5 = convertMD5(res.contentSettings.contentMD5); - assert.strictEqual(convertedMD5, expectedMD5); - return cb(); - }, - err => { - assert.ok(expected.error); - assert.strictEqual(err.statusCode, 404); - assert.strictEqual(err.code, 'NotFound'); - return cb(); - }); + azureClient + .getContainerClient(container) + .getProperties(key) + .then( + res => { + assert.ok(!expected.error); + const convertedMD5 = convertMD5(res.contentSettings.contentMD5); + assert.strictEqual(convertedMD5, expectedMD5); + return cb(); + }, + err => { + assert.ok(expected.error); + assert.strictEqual(err.statusCode, 404); + assert.strictEqual(err.code, 'NotFound'); + return cb(); + }, + ); } -describeSkipIfNotMultiple('Abort MPU on Azure data backend', function -describeF() { +describeSkipIfNotMultiple('Abort MPU on Azure data backend', function describeF() { this.timeout(50000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -52,35 +61,44 @@ describeF() { }); describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(done => { - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) .then(() => done()) .catch(done); }); @@ -92,144 +110,156 @@ describeF() { Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const partParams = { - ...params, - PartNumber: 1, - Body: Buffer.alloc(0), - }; - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => azureCheck(azureContainerName, this.test.key, - expected, next), - ], done); + async.waterfall( + [ + next => { + const partParams = { + ...params, + PartNumber: 1, + Body: Buffer.alloc(0), + }; + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => azureCheck(azureContainerName, this.test.key, expected, next), + ], + done, + ); }); - it('should abort MPU with one part bigger than max subpart', - function itFn(done) { + it('should abort MPU with one part bigger than max subpart', function itFn(done) { const expected = { error: true }; const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const body = Buffer.alloc(maxSubPartSize + 10); - const partParams = { - ...params, - PartNumber: 1, - Body: body, - }; - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => azureCheck(azureContainerName, this.test.key, - expected, next), - ], done); + async.waterfall( + [ + next => { + const body = Buffer.alloc(maxSubPartSize + 10); + const partParams = { + ...params, + PartNumber: 1, + Body: body, + }; + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => azureCheck(azureContainerName, this.test.key, expected, next), + ], + done, + ); }); }); describe('with previously existing object with same key', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - const body = Buffer.alloc(10); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - Body: body, - })) - .then(() => next()) - .catch(err => { - assert.equal(err, null, 'Err putting object to ' + - `azure: ${err}`); - next(err); - }); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': azureLocation, - }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + const body = Buffer.alloc(10); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => { + assert.equal(err, null, 'Err putting object to ' + `azure: ${err}`); + next(err); + }); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': azureLocation, + }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write('Error emptying/deleting bucket: ' + - `${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write('Error emptying/deleting bucket: ' + `${err}\n`); + throw err; + }); }); - it('should abort MPU without deleting existing object', - function itFn(done) { + it('should abort MPU without deleting existing object', function itFn(done) { const expected = { error: false }; const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - const body = Buffer.alloc(10); - const partParams = { - ...params, - PartNumber: 1, - Body: body, - }; - s3.send(new UploadPartCommand(partParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(params)) - .then(() => next()) - .catch(next); - }, - next => azureCheck(azureContainerName, this.test.key, - expected, next), - ], done); + async.waterfall( + [ + next => { + const body = Buffer.alloc(10); + const partParams = { + ...params, + PartNumber: 1, + Body: body, + }; + s3.send(new UploadPartCommand(partParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => next()) + .catch(next); + }, + next => azureCheck(azureContainerName, this.test.key, expected, next), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js index 19e17e48b7..40841a0994 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/azureCompleteMPU.js @@ -1,11 +1,13 @@ const async = require('async'); const assert = require('assert'); -const { CreateBucketCommand, +const { + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, PutObjectCommand, - GetObjectCommand } = require('@aws-sdk/client-s3'); + GetObjectCommand, +} = require('@aws-sdk/client-s3'); const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); @@ -58,64 +60,66 @@ function getCheck(key, bucketMatch, cb) { } function mpuSetup(key, location, cb) { const partArray = []; - async.waterfall([ - next => { - const params = { - Bucket: azureContainerName, - Key: key, - Metadata: { 'scal-location-constraint': location }, - }; - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - const uploadId = res.UploadId; - assert(uploadId); - assert.strictEqual(res.Bucket, azureContainerName); - assert.strictEqual(res.Key, key); - return next(null, uploadId); - }) - .catch(next); - }, - (uploadId, next) => { - const partParams = { - Bucket: azureContainerName, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: smallBody, - }; - s3.send(new UploadPartCommand(partParams)) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 1 }); - return next(null, uploadId); - }) - .catch(next); - }, - (uploadId, next) => { - const partParams = { - Bucket: azureContainerName, - Key: key, - PartNumber: 2, - UploadId: uploadId, - Body: bigBody, - }; - s3.send(new UploadPartCommand(partParams)) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 2 }); - return next(null, uploadId); - }) - .catch(next); + async.waterfall( + [ + next => { + const params = { + Bucket: azureContainerName, + Key: key, + Metadata: { 'scal-location-constraint': location }, + }; + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + const uploadId = res.UploadId; + assert(uploadId); + assert.strictEqual(res.Bucket, azureContainerName); + assert.strictEqual(res.Key, key); + return next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + const partParams = { + Bucket: azureContainerName, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: smallBody, + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 1 }); + return next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + const partParams = { + Bucket: azureContainerName, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: bigBody, + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 2 }); + return next(null, uploadId); + }) + .catch(next); + }, + ], + (err, uploadId) => { + if (err) { + return cb(err); + } + process.stdout.write('Created MPU and put two parts\n'); + return cb(uploadId, partArray); }, - ], (err, uploadId) => { - if (err) { - return cb(err); - } - process.stdout.write('Created MPU and put two parts\n'); - return cb(uploadId, partArray); - }); + ); } -describeSkipIfNotMultiple('Complete MPU API for Azure data backend', -function testSuite() { +describeSkipIfNotMultiple('Complete MPU API for Azure data backend', function testSuite() { this.timeout(150000); withV4(sigCfg => { beforeEach(function beFn() { @@ -123,8 +127,7 @@ function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; this.currentTest.awsClient = awsS3; - return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -132,15 +135,16 @@ function testSuite() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should complete an MPU on Azure', function itFn(done) { @@ -153,17 +157,14 @@ function testSuite() { }; s3.send(new CompleteMultipartUploadCommand(params)) .then(() => { - setTimeout(() => getCheck(this.test.key, true, done), - azureTimeout); + setTimeout(() => getCheck(this.test.key, true, done), azureTimeout); }) .catch(done); }); }); - it('should complete an MPU on Azure with bucketMatch=false', - function itFn(done) { - mpuSetup(this.test.key, azureLocationMismatch, - (uploadId, partArray) => { + it('should complete an MPU on Azure with bucketMatch=false', function itFn(done) { + mpuSetup(this.test.key, azureLocationMismatch, (uploadId, partArray) => { const params = { Bucket: azureContainerName, Key: this.test.key, @@ -172,79 +173,81 @@ function testSuite() { }; s3.send(new CompleteMultipartUploadCommand(params)) .then(() => { - setTimeout(() => getCheck(this.test.key, false, done), - azureTimeout); + setTimeout(() => getCheck(this.test.key, false, done), azureTimeout); }) .catch(done); }); }); - it('should complete an MPU on Azure with same key as object put ' + - 'to file', function itFn(done) { + it('should complete an MPU on Azure with same key as object put ' + 'to file', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } })).then(() => { - mpuSetup(this.test.key, azureLocation, - (uploadId, partArray) => { - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: uploadId, - MultipartUpload: { Parts: partArray }, - }; - s3.send(new CompleteMultipartUploadCommand(params)) - .then(() => { - setTimeout(() => getCheck(this.test.key, true, done), - azureTimeout); - }) - .catch(done); - }); - }).catch(done); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }), + ) + .then(() => { + mpuSetup(this.test.key, azureLocation, (uploadId, partArray) => { + const params = { + Bucket: azureContainerName, + Key: this.test.key, + UploadId: uploadId, + MultipartUpload: { Parts: partArray }, + }; + s3.send(new CompleteMultipartUploadCommand(params)) + .then(() => { + setTimeout(() => getCheck(this.test.key, true, done), azureTimeout); + }) + .catch(done); + }); + }) + .catch(done); }); - it('should complete an MPU on Azure with same key as object put ' + - 'to Azure', function itFn(done) { + it('should complete an MPU on Azure with same key as object put ' + 'to Azure', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': azureLocation } })).then(() => { - mpuSetup(this.test.key, azureLocation, - (uploadId, partArray) => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ).then(() => { + mpuSetup(this.test.key, azureLocation, (uploadId, partArray) => { const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: uploadId, MultipartUpload: { Parts: partArray }, }; - s3.send(new CompleteMultipartUploadCommand(params)).then(() => { - - setTimeout(() => getCheck(this.test.key, true, done), - azureTimeout); - }).catch(err => { - assert.equal(err, null, `Err completing MPU: ${err}`); - done(err); - }); + s3.send(new CompleteMultipartUploadCommand(params)) + .then(() => { + setTimeout(() => getCheck(this.test.key, true, done), azureTimeout); + }) + .catch(err => { + assert.equal(err, null, `Err completing MPU: ${err}`); + done(err); + }); }); }); }); - it('should complete an MPU on Azure with same key as object put ' + - 'to AWS', function itFn(done) { + it('should complete an MPU on Azure with same key as object put ' + 'to AWS', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } - })) + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, azureLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, azureLocation, (uploadId, partArray) => { const params = { Bucket: azureContainerName, Key: this.test.key, @@ -255,10 +258,13 @@ function testSuite() { .then(() => { // make sure object is gone from AWS setTimeout(() => { - this.test.awsClient.send(new GetObjectCommand({ - Bucket: awsBucket, - Key: this.test.key - })) + this.test.awsClient + .send( + new GetObjectCommand({ + Bucket: awsBucket, + Key: this.test.key, + }), + ) .then(() => { done(new Error('Expected NoSuchKey error')); }) diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js index 0b346fb4cb..624a590c40 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/completeMPUGcp.js @@ -11,9 +11,18 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { fileLocation, awsS3, awsLocation, - awsBucket, gcpClient, gcpBucket, gcpLocation, gcpLocationMismatch, - genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { + fileLocation, + awsS3, + awsLocation, + awsBucket, + gcpClient, + gcpBucket, + gcpLocation, + gcpLocationMismatch, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const bucket = `completempugcp${genUniqID()}`; const smallBody = Buffer.from('I am a body', 'utf8'); @@ -28,10 +37,12 @@ let bucketUtil; function getCheck(key, bucketMatch, cb) { (async () => { let gcpKey = key; - const s3Res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: gcpKey, - })); + const s3Res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: gcpKey, + }), + ); assert.strictEqual(s3Res.ETag, `"${s3MD5}"`); if (!bucketMatch) { @@ -52,59 +63,67 @@ function getCheck(key, bucketMatch, cb) { function mpuSetup(key, location, cb) { const partArray = []; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Metadata: { 'scal-location-constraint': location }, - })) - .then(res => { - const uploadId = res.UploadId; - assert(uploadId); - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, key); - next(null, uploadId); - }) - .catch(next); + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Metadata: { 'scal-location-constraint': location }, + }), + ) + .then(res => { + const uploadId = res.UploadId; + assert(uploadId); + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, key); + next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: smallBody, + }), + ) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 1 }); + next(null, uploadId); + }) + .catch(next); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: bigBody, + }), + ) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 2 }); + next(null, uploadId); + }) + .catch(next); + }, + ], + (err, uploadId) => { + process.stdout.write('Created MPU and put two parts\n'); + assert.equal(err, null, `Err setting up MPU: ${err}`); + cb(uploadId, partArray); }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: smallBody, - })) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 1 }); - next(null, uploadId); - }) - .catch(next); - }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 2, - UploadId: uploadId, - Body: bigBody, - })) - .then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 2 }); - next(null, uploadId); - }) - .catch(next); - }, - ], (err, uploadId) => { - process.stdout.write('Created MPU and put two parts\n'); - assert.equal(err, null, `Err setting up MPU: ${err}`); - cb(uploadId, partArray); - }); + ); } -describeSkipIfNotMultiple('Complete MPU API for GCP data backend', -function testSuite() { +describeSkipIfNotMultiple('Complete MPU API for GCP data backend', function testSuite() { this.timeout(150000); withV4(sigCfg => { beforeEach(function beFn() { @@ -112,24 +131,24 @@ function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; this.currentTest.awsClient = awsS3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }); + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should complete an MPU on GCP', function itFn(done) { @@ -148,10 +167,8 @@ function testSuite() { }); }); - it('should complete an MPU on GCP with bucketMatch=false', - function itFn(done) { - mpuSetup(this.test.key, gcpLocationMismatch, - (uploadId, partArray) => { + it('should complete an MPU on GCP with bucketMatch=false', function itFn(done) { + mpuSetup(this.test.key, gcpLocationMismatch, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -161,23 +178,23 @@ function testSuite() { setTimeout(() => { s3.send(new CompleteMultipartUploadCommand(params)) .then(() => getCheck(this.test.key, false, done)) - .catch(done); + .catch(done); }, gcpTimeout); }); }); - it('should complete an MPU on GCP with same key as object put ' + - 'to file', function itFn(done) { + it('should complete an MPU on GCP with same key as object put ' + 'to file', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, gcpLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, gcpLocation, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -197,18 +214,18 @@ function testSuite() { }); }); - it('should complete an MPU on GCP with same key as object put ' + - 'to GCP', function itFn(done) { + it('should complete an MPU on GCP with same key as object put ' + 'to GCP', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, gcpLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, gcpLocation, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -228,18 +245,18 @@ function testSuite() { }); }); - it('should complete an MPU on GCP with same key as object put ' + - 'to AWS', function itFn(done) { + it('should complete an MPU on GCP with same key as object put ' + 'to AWS', function itFn(done) { const body = Buffer.from('I am a body', 'utf8'); - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.test.key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.test.key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) .then(() => { - mpuSetup(this.test.key, gcpLocation, - (uploadId, partArray) => { + mpuSetup(this.test.key, gcpLocation, (uploadId, partArray) => { const params = { Bucket: bucket, Key: this.test.key, @@ -250,8 +267,7 @@ function testSuite() { .then(() => { // make sure object is gone from AWS setTimeout(() => { - this.test.awsClient.getObject({ Bucket: awsBucket, - Key: this.test.key }, err => { + this.test.awsClient.getObject({ Bucket: awsBucket, Key: this.test.key }, err => { assert.strictEqual(err.code, 'NoSuchKey'); getCheck(this.test.key, true, done); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js index 766208c85f..927ac88a7c 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuComplete/mpuAwsVersioning.js @@ -2,9 +2,14 @@ const assert = require('assert'); const async = require('async'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { CreateBucketCommand, DeleteBucketCommand, - CreateMultipartUploadCommand, UploadPartCommand, - CompleteMultipartUploadCommand, DeleteObjectCommand } = require('@aws-sdk/client-s3'); +const { + CreateBucketCommand, + DeleteBucketCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, + DeleteObjectCommand, +} = require('@aws-sdk/client-s3'); const { minimumAllowedPartSize } = require('../../../../../../constants'); const { removeAllVersions } = require('../../../lib/utility/versioning-util'); const { @@ -25,161 +30,195 @@ const bucket = `mpuawsversioning${genUniqID()}`; function mpuSetup(s3, key, location, cb) { const partArray = []; - async.waterfall([ - next => { - const params = { - Bucket: bucket, - Key: key, - Metadata: { 'scal-location-constraint': location }, - }; - s3.send(new CreateMultipartUploadCommand(params)).then(res => { - const uploadId = res.UploadId; - assert(uploadId); - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, key); - next(null, uploadId); - }).catch(err => { - next(err); - }); - }, - (uploadId, next) => { - const partParams = { - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: data[0], - }; - s3.send(new UploadPartCommand(partParams)).then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 1 }); - next(null, uploadId); - }).catch(err => { - next(err); - }); - }, - (uploadId, next) => { - const partParams = { - Bucket: bucket, - Key: key, - PartNumber: 2, - UploadId: uploadId, - Body: data[1], - }; - s3.send(new UploadPartCommand(partParams)).then(res => { - partArray.push({ ETag: res.ETag, PartNumber: 2 }); - next(null, uploadId); - }).catch(err => { - next(err); - }); + async.waterfall( + [ + next => { + const params = { + Bucket: bucket, + Key: key, + Metadata: { 'scal-location-constraint': location }, + }; + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + const uploadId = res.UploadId; + assert(uploadId); + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, key); + next(null, uploadId); + }) + .catch(err => { + next(err); + }); + }, + (uploadId, next) => { + const partParams = { + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: data[0], + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 1 }); + next(null, uploadId); + }) + .catch(err => { + next(err); + }); + }, + (uploadId, next) => { + const partParams = { + Bucket: bucket, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: data[1], + }; + s3.send(new UploadPartCommand(partParams)) + .then(res => { + partArray.push({ ETag: res.ETag, PartNumber: 2 }); + next(null, uploadId); + }) + .catch(err => { + next(err); + }); + }, + ], + (err, uploadId) => { + process.stdout.write('Created MPU and put two parts\n'); + cb(err, uploadId, partArray); }, - ], (err, uploadId) => { - process.stdout.write('Created MPU and put two parts\n'); - cb(err, uploadId, partArray); - }); + ); } function completeAndAssertMpu(s3, params, cb) { - const { bucket, key, uploadId, partArray, expectVersionId, - expectedGetVersionId } = params; - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { Parts: partArray }, - })).then(data => { - if (expectVersionId) { - assert.notEqual(data.VersionId, undefined); - } else { - assert.strictEqual(data.VersionId, undefined); - } - const expectedVersionId = expectedGetVersionId || data.VersionId; - getAndAssertResult(s3, { bucket, key, body: concattedData, - expectedVersionId }, cb); - }).catch(err => { - cb(err); - }); + const { bucket, key, uploadId, partArray, expectVersionId, expectedGetVersionId } = params; + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { Parts: partArray }, + }), + ) + .then(data => { + if (expectVersionId) { + assert.notEqual(data.VersionId, undefined); + } else { + assert.strictEqual(data.VersionId, undefined); + } + const expectedVersionId = expectedGetVersionId || data.VersionId; + getAndAssertResult(s3, { bucket, key, body: concattedData, expectedVersionId }, cb); + }) + .catch(err => { + cb(err); + }); } -describeSkipIfNotMultiple('AWS backend complete mpu with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend complete mpu with versioning', function testSuite() { this.timeout(120000); withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - beforeEach(done => s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })).then(() => done()).catch(err => done(err))); + beforeEach(done => + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ) + .then(() => done()) + .catch(err => done(err)), + ); afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); - it('versioning not configured: should not return version id ' + - 'completing mpu', done => { + it('versioning not configured: should not return version id ' + 'completing mpu', done => { const key = `somekey-${genUniqID()}`; mpuSetup(s3, key, awsLocation, (err, uploadId, partArray) => { - completeAndAssertMpu(s3, { bucket, key, uploadId, partArray, - expectVersionId: false }, done); + completeAndAssertMpu(s3, { bucket, key, uploadId, partArray, expectVersionId: false }, done); }); }); - it('versioning not configured: if complete mpu on already-existing ' + - 'object, metadata should be overwritten but data of previous version' + - 'in AWS should not be deleted', function itF(done) { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putToAwsBackend(s3, bucket, key, '', err => next(err)), - next => awsGetLatestVerId(key, '', next), - (awsVerId, next) => { - this.test.awsVerId = awsVerId; - next(); - }, - next => mpuSetup(s3, key, awsLocation, next), - (uploadId, partArray, next) => completeAndAssertMpu(s3, - { bucket, key, uploadId, partArray, expectVersionId: - false }, next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: - 'null' })).then(delData => next(null, delData)).catch(next), - (delData, next) => getAndAssertResult(s3, { bucket, key, - expectedError: 'NoSuchKey' }, next), - next => awsGetLatestVerId(key, '', next), - (awsVerId, next) => { - assert.strictEqual(awsVerId, this.test.awsVerId); - next(); - }, - ], done); - }); + it( + 'versioning not configured: if complete mpu on already-existing ' + + 'object, metadata should be overwritten but data of previous version' + + 'in AWS should not be deleted', + function itF(done) { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putToAwsBackend(s3, bucket, key, '', err => next(err)), + next => awsGetLatestVerId(key, '', next), + (awsVerId, next) => { + this.test.awsVerId = awsVerId; + next(); + }, + next => mpuSetup(s3, key, awsLocation, next), + (uploadId, partArray, next) => + completeAndAssertMpu( + s3, + { bucket, key, uploadId, partArray, expectVersionId: false }, + next, + ), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: 'null' })) + .then(delData => next(null, delData)) + .catch(next), + (delData, next) => getAndAssertResult(s3, { bucket, key, expectedError: 'NoSuchKey' }, next), + next => awsGetLatestVerId(key, '', next), + (awsVerId, next) => { + assert.strictEqual(awsVerId, this.test.awsVerId); + next(); + }, + ], + done, + ); + }, + ); - it('versioning suspended: should not return version id completing mpu', - done => { + it('versioning suspended: should not return version id completing mpu', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => suspendVersioning(s3, bucket, next), - next => mpuSetup(s3, key, awsLocation, next), - (uploadId, partArray, next) => completeAndAssertMpu(s3, - { bucket, key, uploadId, partArray, expectVersionId: false, - expectedGetVersionId: 'null' }, next), - ], done); + async.waterfall( + [ + next => suspendVersioning(s3, bucket, next), + next => mpuSetup(s3, key, awsLocation, next), + (uploadId, partArray, next) => + completeAndAssertMpu( + s3, + { bucket, key, uploadId, partArray, expectVersionId: false, expectedGetVersionId: 'null' }, + next, + ), + ], + done, + ); }); - it('versioning enabled: should return version id completing mpu', - done => { + it('versioning enabled: should return version id completing mpu', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => mpuSetup(s3, key, awsLocation, next), - (uploadId, partArray, next) => completeAndAssertMpu(s3, - { bucket, key, uploadId, partArray, expectVersionId: true }, - next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => mpuSetup(s3, key, awsLocation, next), + (uploadId, partArray, next) => + completeAndAssertMpu(s3, { bucket, key, uploadId, partArray, expectVersionId: true }, next), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js index f342472f90..4d80bfad68 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/azurePutPart.js @@ -14,10 +14,16 @@ const { const { s3middleware } = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { expectedETag, uniqName, getAzureClient, - getAzureContainerName, convertMD5, azureLocation, azureLocationMismatch, - describeSkipIfNotMultiple } - = require('../utils'); +const { + expectedETag, + uniqName, + getAzureClient, + getAzureContainerName, + convertMD5, + azureLocation, + azureLocationMismatch, + describeSkipIfNotMultiple, +} = require('../utils'); const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const maxSubPartSize = azureMpuUtils.maxSubPartSize; const getBlockId = azureMpuUtils.getBlockId; @@ -31,15 +37,19 @@ let bucketUtil; let s3; function checkSubPart(key, uploadId, expectedParts, cb) { - azureClient.getContainerClient(azureContainerName) + azureClient + .getContainerClient(azureContainerName) .getBlockBlobClient(key) - .getBlockList('all').then(list => { + .getBlockList('all') + .then(list => { const uncommittedBlocks = list.uncommittedBlocks; const committedBlocks = list.committedBlocks; assert.strictEqual(committedBlocks, undefined); uncommittedBlocks.forEach((l, index) => { - assert.strictEqual(l.name, getBlockId(uploadId, - expectedParts[index].partnbr, expectedParts[index].subpartnbr)); + assert.strictEqual( + l.name, + getBlockId(uploadId, expectedParts[index].partnbr, expectedParts[index].subpartnbr), + ); assert.strictEqual(l.size, expectedParts[index].size.toString()); }); cb(); @@ -48,14 +58,15 @@ function checkSubPart(key, uploadId, expectedParts, cb) { function azureCheck(key, cb) { (async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: key, - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: key, + }), + ); assert(res, 'expected getObject response'); assert.strictEqual(res.ETag, `"${expectedMD5}"`); - const properties = await azureClient.getContainerClient(azureContainerName) - .getProperties(key); + const properties = await azureClient.getContainerClient(azureContainerName).getProperties(key); const convertedMD5 = convertMD5(properties.contentSettings.contentMD5); assert.strictEqual(convertedMD5, expectedMD5); })() @@ -63,8 +74,7 @@ function azureCheck(key, cb) { .catch(err => cb(err)); } -describeSkipIfNotMultiple('MultipleBackend put part to AZURE', function -describeF() { +describeSkipIfNotMultiple('MultipleBackend put part to AZURE', function describeF() { this.timeout(80000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -74,51 +84,65 @@ describeF() { }); describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); - }); + ); }); it('should put 0-byte block to Azure', function itFn(done) { @@ -128,35 +152,44 @@ describeF() { UploadId: this.test.uploadId, PartNumber: 1, }; - async.waterfall([ - next => { - const uploadParams = { - ...params, - Body: Buffer.alloc(0), - }; - s3.send(new UploadPartCommand(uploadParams)) - .then(res => { - const eTagExpected = `"${azureMpuUtils.zeroByteETag}"`; - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => azureClient.getContainerClient(azureContainerName) - .getBlockBlobClient(this.test.key) - .getBlockList('all').then( - () => assert.fail('Expected failure but got success'), err => { - assert.strictEqual(err.code, 'BlobNotFound'); - next(); - }), - ], done); + async.waterfall( + [ + next => { + const uploadParams = { + ...params, + Body: Buffer.alloc(0), + }; + s3.send(new UploadPartCommand(uploadParams)) + .then(res => { + const eTagExpected = `"${azureMpuUtils.zeroByteETag}"`; + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => + azureClient + .getContainerClient(azureContainerName) + .getBlockBlobClient(this.test.key) + .getBlockList('all') + .then( + () => assert.fail('Expected failure but got success'), + err => { + assert.strictEqual(err.code, 'BlobNotFound'); + next(); + }, + ), + ], + done, + ); }); it('should put 2 blocks to Azure', function itFn(done) { const body = Buffer.alloc(maxSubPartSize + 10); - const parts = [{ partnbr: 1, subpartnbr: 0, - size: maxSubPartSize }, - { partnbr: 1, subpartnbr: 1, size: 10 }]; + const parts = [ + { partnbr: 1, subpartnbr: 0, size: maxSubPartSize }, + { partnbr: 1, subpartnbr: 1, size: 10 }, + ]; const params = { Bucket: azureContainerName, Key: this.test.key, @@ -164,218 +197,244 @@ describeF() { PartNumber: 1, Body: body, }; - async.waterfall([ - next => { - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = expectedETag(body); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => checkSubPart(this.test.key, this.test.uploadId, - parts, next), - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCommand(params)) + .then(res => { + const eTagExpected = expectedETag(body); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => checkSubPart(this.test.key, this.test.uploadId, parts, next), + ], + done, + ); }); - it('should put 5 parts bigger than maxSubPartSize to Azure', - function it(done) { + it('should put 5 parts bigger than maxSubPartSize to Azure', function it(done) { const body = Buffer.alloc(maxSubPartSize + 10); let parts = []; for (let i = 1; i < 6; i++) { parts = parts.concat([ - { partnbr: i, subpartnbr: 0, size: maxSubPartSize }, - { partnbr: i, subpartnbr: 1, size: 10 }, + { partnbr: i, subpartnbr: 0, size: maxSubPartSize }, + { partnbr: i, subpartnbr: 1, size: 10 }, ]); } - async.times(5, (n, next) => { - const partNumber = n + 1; - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: partNumber, - Body: body, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = expectedETag(body); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, err => { - assert.equal(err, null, 'Expected success, ' + - `got error: ${err}`); - checkSubPart(this.test.key, this.test.uploadId, - parts, done); - }); + async.times( + 5, + (n, next) => { + const partNumber = n + 1; + const params = { + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: partNumber, + Body: body, + }; + s3.send(new UploadPartCommand(params)) + .then(res => { + const eTagExpected = expectedETag(body); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + err => { + assert.equal(err, null, 'Expected success, ' + `got error: ${err}`); + checkSubPart(this.test.key, this.test.uploadId, parts, done); + }, + ); }); - it('should put 5 parts smaller than maxSubPartSize to Azure', - function it(done) { + it('should put 5 parts smaller than maxSubPartSize to Azure', function it(done) { const body = Buffer.alloc(10); let parts = []; for (let i = 1; i < 6; i++) { - parts = parts.concat([ - { partnbr: i, subpartnbr: 0, size: 10 }, - ]); + parts = parts.concat([{ partnbr: i, subpartnbr: 0, size: 10 }]); } - async.times(5, (n, next) => { - const partNumber = n + 1; - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: partNumber, - Body: body, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = expectedETag(body); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, err => { - assert.equal(err, null, 'Expected success, ' + - `got error: ${err}`); - checkSubPart(this.test.key, this.test.uploadId, - parts, done); - }); - }); - - it('should put the same part twice', function itFn(done) { - const body1 = Buffer.alloc(maxSubPartSize + 10); - const body2 = Buffer.alloc(20); - const parts2 = [{ partnbr: 1, subpartnbr: 0, size: 20 }, - { partnbr: 1, subpartnbr: 1, size: 10 }]; - async.waterfall([ - next => { - s3.send(new UploadPartCommand({ - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body1, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new UploadPartCommand({ + async.times( + 5, + (n, next) => { + const partNumber = n + 1; + const params = { Bucket: azureContainerName, Key: this.test.key, UploadId: this.test.uploadId, - PartNumber: 1, - Body: body2, - })) + PartNumber: partNumber, + Body: body, + }; + s3.send(new UploadPartCommand(params)) .then(res => { - const eTagExpected = expectedETag(body2); + const eTagExpected = expectedETag(body); assert.strictEqual(res.ETag, eTagExpected); next(); }) .catch(next); }, - next => checkSubPart(this.test.key, this.test.uploadId, - parts2, next), - ], done); + err => { + assert.equal(err, null, 'Expected success, ' + `got error: ${err}`); + checkSubPart(this.test.key, this.test.uploadId, parts, done); + }, + ); + }); + + it('should put the same part twice', function itFn(done) { + const body1 = Buffer.alloc(maxSubPartSize + 10); + const body2 = Buffer.alloc(20); + const parts2 = [ + { partnbr: 1, subpartnbr: 0, size: 20 }, + { partnbr: 1, subpartnbr: 1, size: 10 }, + ]; + async.waterfall( + [ + next => { + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body1, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body2, + }), + ) + .then(res => { + const eTagExpected = expectedETag(body2); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => checkSubPart(this.test.key, this.test.uploadId, parts2, next), + ], + done, + ); }); }); describe('with same key as preexisting part', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - const body = Buffer.alloc(10); - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': - azureLocation }, - Body: body, - })) - .then(() => next()) - .catch(err => { - assert.equal(err, null, 'Err putting object to ' + - `azure: ${err}`); - next(err); - }); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + const body = Buffer.alloc(10); + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => { + assert.equal(err, null, 'Err putting object to ' + `azure: ${err}`); + next(err); + }); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - process.stdout.write('Aborting multipart upload\n'); - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - process.stdout.write('Deleting object\n'); - s3.send(new DeleteObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - })) - .then(() => next()) - .catch(next); - }, - next => { - process.stdout.write('Deleting bucket\n'); - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + process.stdout.write('Aborting multipart upload\n'); + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting object\n'); + s3.send( + new DeleteObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting bucket\n'); + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Err in afterEach: ${err}`); + done(); }, - ], err => { - assert.equal(err, null, `Err in afterEach: ${err}`); - done(); - }); + ); }); - it('should put a part without overwriting existing object', - function itFn(done) { + it('should put a part without overwriting existing object', function itFn(done) { const body = Buffer.alloc(20); - s3.send(new UploadPartCommand({ - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body, - })) + s3.send( + new UploadPartCommand({ + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body, + }), + ) .then(() => { azureCheck(this.test.key, done); }) .catch(err => { - assert.strictEqual(err, null, 'Err putting part to ' + - `Azure: ${err}`); + assert.strictEqual(err, null, 'Err putting part to ' + `Azure: ${err}`); done(err); }); }); @@ -383,94 +442,107 @@ describeF() { }); }); -describeSkipIfNotMultiple('MultipleBackend put part to AZURE ' + -'location with bucketMatch sets to false', function -describeF() { - this.timeout(80000); - withV4(sigCfg => { - beforeEach(function beforeFn() { - this.currentTest.key = uniqName(keyObject); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - }); - describe('with bucket location header', () => { - beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); +describeSkipIfNotMultiple( + 'MultipleBackend put part to AZURE ' + 'location with bucketMatch sets to false', + function describeF() { + this.timeout(80000); + withV4(sigCfg => { + beforeEach(function beforeFn() { + this.currentTest.key = uniqName(keyObject); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; }); + describe('with bucket location header', () => { + beforeEach(function beforeEachFn(done) { + async.waterfall( + [ + next => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); + }); - afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ - Bucket: azureContainerName, - })) - .then(() => next()) - .catch(next); - }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); + afterEach(function afterEachFn(done) { + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new DeleteBucketCommand({ + Bucket: azureContainerName, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); + }, + ); }); - }); - it('should put block to AZURE location with bucketMatch' + - ' sets to false', function itFn(done) { - const body20 = Buffer.alloc(20); - const params = { - Bucket: azureContainerName, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body20, - }; - const parts = [{ partnbr: 1, subpartnbr: 0, - size: 20 }]; - async.waterfall([ - next => { - s3.send(new UploadPartCommand(params)) - .then(res => { - const eTagExpected = - '"441018525208457705bf09a8ee3c1093"'; - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => checkSubPart( - `${azureContainerName}/${this.test.key}`, - this.test.uploadId, parts, next), - ], done); + it('should put block to AZURE location with bucketMatch' + ' sets to false', function itFn(done) { + const body20 = Buffer.alloc(20); + const params = { + Bucket: azureContainerName, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body20, + }; + const parts = [{ partnbr: 1, subpartnbr: 0, size: 20 }]; + async.waterfall( + [ + next => { + s3.send(new UploadPartCommand(params)) + .then(res => { + const eTagExpected = '"441018525208457705bf09a8ee3c1093"'; + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => + checkSubPart(`${azureContainerName}/${this.test.key}`, this.test.uploadId, parts, next), + ], + done, + ); + }); }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js index cbd8222074..03c8d411a7 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/mpuParts/putPartGcp.js @@ -13,10 +13,16 @@ const arsenal = require('arsenal'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucket, gcpBucketMPU, - gcpLocation, gcpLocationMismatch, uniqName, genUniqID, - describeSkipIfNotMultiple } - = require('../utils'); +const { + gcpClient, + gcpBucket, + gcpBucketMPU, + gcpLocation, + gcpLocationMismatch, + uniqName, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const { createMpuKey } = arsenal.storage.data.external.GcpUtils; const keyObject = 'putgcp'; @@ -35,20 +41,16 @@ function checkMPUResult(bucket, key, uploadId, objCount, expected, cb) { UploadId: uploadId, }; gcpClient.listParts(params, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert((res && res.Contents && - res.Contents.length === objCount)); + assert.ifError(err, `Expected success, but got err ${err}`); + assert(res && res.Contents && res.Contents.length === objCount); res.Contents.forEach(part => { - assert.strictEqual( - part.ETag, `"${expected}"`); + assert.strictEqual(part.ETag, `"${expected}"`); }); cb(); }); } -describeSkipIfNotMultiple('MultipleBacked put part to GCP', function -describeFn() { +describeSkipIfNotMultiple('MultipleBacked put part to GCP', function describeFn() { this.timeout(180000); withV4(sigCfg => { beforeEach(function beforeFn() { @@ -59,47 +61,57 @@ describeFn() { describe('with bucket location header', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); - }); + ); }); it('should put 0-byte part to GCP', function itFn(done) { @@ -109,282 +121,320 @@ describeFn() { UploadId: this.test.uploadId, PartNumber: 1, }; - async.waterfall([ - next => { - s3.send(new UploadPartCommand({ - ...params, - Body: Buffer.alloc(0), - })) - .then(res => { + async.waterfall( + [ + next => { + s3.send( + new UploadPartCommand({ + ...params, + Body: Buffer.alloc(0), + }), + ) + .then(res => { + assert.strictEqual(res.ETag, `"${emptyMD5}"`); + next(); + }) + .catch(next); + }, + next => { + const mpuKey = createMpuKey(this.test.key, this.test.uploadId, 1); + const getParams = { + Bucket: gcpBucketMPU, + Key: mpuKey, + }; + gcpClient.getObject(getParams, (err, res) => { + assert.ifError(err, `Expected success, but got err ${err}`); assert.strictEqual(res.ETag, `"${emptyMD5}"`); next(); - }) - .catch(next); - }, - next => { - const mpuKey = - createMpuKey(this.test.key, this.test.uploadId, 1); - const getParams = { - Bucket: gcpBucketMPU, - Key: mpuKey, - }; - gcpClient.getObject(getParams, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.strictEqual(res.ETag, `"${emptyMD5}"`); - next(); - }); - }, - ], done); + }); + }, + ], + done, + ); }); it('should put 2 parts to GCP', function ifFn(done) { - async.waterfall([ - next => { - async.times(2, (n, cb) => { - const params = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - Body: body, - PartNumber: n + 1, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - assert.strictEqual( - res.ETag, `"${correctMD5}"`); - cb(); - }) - .catch(cb); - }, err => next(err)); - }, - next => checkMPUResult( - gcpBucketMPU, this.test.key, this.test.uploadId, - 2, correctMD5, next), - ], done); + async.waterfall( + [ + next => { + async.times( + 2, + (n, cb) => { + const params = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + Body: body, + PartNumber: n + 1, + }; + s3.send(new UploadPartCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${correctMD5}"`); + cb(); + }) + .catch(cb); + }, + err => next(err), + ); + }, + next => checkMPUResult(gcpBucketMPU, this.test.key, this.test.uploadId, 2, correctMD5, next), + ], + done, + ); }); it('should put the same part twice', function ifFn(done) { - async.waterfall([ - next => { - const partBody = ['', body]; - const partMD5 = [emptyMD5, correctMD5]; - async.timesSeries(2, (n, cb) => { - const params = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - Body: partBody[n], - PartNumber: 1, - }; - s3.send(new UploadPartCommand(params)) - .then(res => { - assert.strictEqual( - res.ETag, `"${partMD5[n]}"`); - cb(); - }) - .catch(cb); - }, err => next(err)); - }, - next => checkMPUResult( - gcpBucketMPU, this.test.key, this.test.uploadId, - 1, correctMD5, next), - ], done); + async.waterfall( + [ + next => { + const partBody = ['', body]; + const partMD5 = [emptyMD5, correctMD5]; + async.timesSeries( + 2, + (n, cb) => { + const params = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + Body: partBody[n], + PartNumber: 1, + }; + s3.send(new UploadPartCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${partMD5[n]}"`); + cb(); + }) + .catch(cb); + }, + err => next(err), + ); + }, + next => checkMPUResult(gcpBucketMPU, this.test.key, this.test.uploadId, 1, correctMD5, next), + ], + done, + ); }); }); describe('with same key as preexisting part', () => { beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { - 'scal-location-constraint': gcpLocation }, - Body: body, - })) - .then(() => next()) - .catch(err => next(new Error( - `Err putting object to GCP: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { + 'scal-location-constraint': gcpLocation, + }, + Body: body, + }), + ) + .then(() => next()) + .catch(err => next(new Error(`Err putting object to GCP: ${err}`))); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); }); afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - process.stdout.write('Aborting multipart upload\n'); - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - process.stdout.write('Deleting object\n'); - s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: this.currentTest.key, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + process.stdout.write('Aborting multipart upload\n'); + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting object\n'); + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: this.currentTest.key, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + process.stdout.write('Deleting bucket\n'); + s3.send( + new DeleteBucketCommand({ + Bucket: bucket, + }), + ) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Err in afterEach: ${err}`); + done(); }, - next => { - process.stdout.write('Deleting bucket\n'); - s3.send(new DeleteBucketCommand({ - Bucket: bucket, - })) - .then(() => next()) - .catch(next); - }, - ], err => { - assert.equal(err, null, `Err in afterEach: ${err}`); - done(); - }); + ); }); - it('should put a part without overwriting existing object', - function itFn(done) { + it('should put a part without overwriting existing object', function itFn(done) { const body = Buffer.alloc(20); - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body, - })) + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body, + }), + ) .then(() => { - gcpClient.getObject({ - Bucket: gcpBucket, - Key: this.test.key, - }, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - done(); - }); + gcpClient.getObject( + { + Bucket: gcpBucket, + Key: this.test.key, + }, + (err, res) => { + assert.ifError(err, `Expected success, but got err ${err}`); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); + }, + ); }) - .catch(err => done(new Error( - `Err putting part to GCP: ${err}`))); + .catch(err => done(new Error(`Err putting part to GCP: ${err}`))); }); }); }); }); -describeSkipIfNotMultiple('MultipleBackend put part to GCP location ' + -'with bucketMatch sets to false', function -describeF() { - this.timeout(80000); - withV4(sigCfg => { - beforeEach(function beforeFn() { - this.currentTest.key = uniqName(keyObject); - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - }); - describe('with bucket location header', () => { - beforeEach(function beforeEachFn(done) { - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - Metadata: { 'scal-location-constraint': - gcpLocationMismatch }, - })) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(next); - }, - ], done); +describeSkipIfNotMultiple( + 'MultipleBackend put part to GCP location ' + 'with bucketMatch sets to false', + function describeF() { + this.timeout(80000); + withV4(sigCfg => { + beforeEach(function beforeFn() { + this.currentTest.key = uniqName(keyObject); + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; }); + describe('with bucket location header', () => { + beforeEach(function beforeEachFn(done) { + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }), + ) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(next); + }, + ], + done, + ); + }); - afterEach(function afterEachFn(done) { - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: this.currentTest.key, - UploadId: this.currentTest.uploadId, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - ], err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - done(); + afterEach(function afterEachFn(done) { + async.waterfall( + [ + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: this.currentTest.key, + UploadId: this.currentTest.uploadId, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + done(); + }, + ); }); - }); - it('should put part to GCP location with bucketMatch' + - ' sets to false', function itFn(done) { - const body20 = Buffer.alloc(20); - const params = { - Bucket: bucket, - Key: this.test.key, - UploadId: this.test.uploadId, - PartNumber: 1, - Body: body20, - }; - const eTagExpected = - '"441018525208457705bf09a8ee3c1093"'; - async.waterfall([ - next => { - s3.send(new UploadPartCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, eTagExpected); - next(); - }) - .catch(next); - }, - next => { - const key = - createMpuKey(this.test.key, this.test.uploadId, 1); - const mpuKey = `${bucket}/${key}`; - const getParams = { - Bucket: gcpBucketMPU, - Key: mpuKey, - }; - gcpClient.getObject(getParams, (err, res) => { - assert.ifError(err, - `Expected success, but got err ${err}`); - assert.strictEqual(res.ETag, eTagExpected); - next(); - }); - }, - ], done); + it('should put part to GCP location with bucketMatch' + ' sets to false', function itFn(done) { + const body20 = Buffer.alloc(20); + const params = { + Bucket: bucket, + Key: this.test.key, + UploadId: this.test.uploadId, + PartNumber: 1, + Body: body20, + }; + const eTagExpected = '"441018525208457705bf09a8ee3c1093"'; + async.waterfall( + [ + next => { + s3.send(new UploadPartCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, eTagExpected); + next(); + }) + .catch(next); + }, + next => { + const key = createMpuKey(this.test.key, this.test.uploadId, 1); + const mpuKey = `${bucket}/${key}`; + const getParams = { + Bucket: gcpBucketMPU, + Key: mpuKey, + }; + gcpClient.getObject(getParams, (err, res) => { + assert.ifError(err, `Expected success, but got err ${err}`); + assert.strictEqual(res.ETag, eTagExpected); + next(); + }); + }, + ], + done, + ); + }); }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js index a2fdfe3cec..2d37353d23 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/azureObjectCopy.js @@ -1,11 +1,6 @@ const assert = require('assert'); const async = require('async'); -const { - CreateBucketCommand, - PutObjectCommand, - GetObjectCommand, - CopyObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, GetObjectCommand, CopyObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); @@ -22,8 +17,7 @@ const { genUniqID, describeSkipIfNotMultiple, } = require('../utils'); -const { createEncryptedBucketPromise } = - require('../../../lib/utility/createEncryptedBucket'); +const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); const azureClient = getAzureClient(); const azureContainerName = getAzureContainerName(azureLocation); @@ -60,7 +54,9 @@ function normalizeMetadata(metadata) { } function putSourceObj(key, location, objSize, bucket, cb) { - const sourceParams = { Bucket: bucket, Key: key, + const sourceParams = { + Bucket: bucket, + Key: key, Metadata: { 'test-header': 'copyme', }, @@ -87,69 +83,84 @@ function putSourceObj(key, location, objSize, bucket, cb) { .catch(err => cb(new Error(`Error putting source object: ${err}`))); } -function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey, -destBucket, destLoc, azureKey, mdDirective, objSize, callback) { +function assertGetObjects( + sourceKey, + sourceBucket, + sourceLoc, + destKey, + destBucket, + destLoc, + azureKey, + mdDirective, + objSize, + callback, +) { const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey }; const destGetParams = { Bucket: destBucket, Key: destKey }; - async.series([ - cb => { - s3.send(new GetObjectCommand(sourceGetParams)) - .then(res => cb(null, res)) - .catch(cb); - }, - cb => { - s3.send(new GetObjectCommand(destGetParams)) - .then(res => cb(null, res)) - .catch(cb); - }, - cb => azureClient.getContainerClient(azureContainerName) - .getProperties(azureKey) - .then(res => cb(null, res), err => cb(err)), - ], (err, results) => { - assert.equal(err, null, `Error in assertGetObjects: ${err}`); - const [sourceRes, destRes, azureRes] = results; - const sourceMetadata = normalizeMetadata(sourceRes.Metadata); - const destMetadata = normalizeMetadata(destRes.Metadata); - const convertedMD5 = convertMD5(azureRes[0].contentSettings.contentMD5); - if (objSize && objSize.empty) { - assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(convertedMD5, `${emptyMD5}`); - assert.strictEqual('0', azureRes[0].contentLength); - } else if (objSize && objSize.big) { - assert.strictEqual(sourceRes.ETag, `"${bigMD5}"`); - assert.strictEqual(destRes.ETag, `"${bigMD5}"`); - if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); - assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + async.series( + [ + cb => { + s3.send(new GetObjectCommand(sourceGetParams)) + .then(res => cb(null, res)) + .catch(cb); + }, + cb => { + s3.send(new GetObjectCommand(destGetParams)) + .then(res => cb(null, res)) + .catch(cb); + }, + cb => + azureClient + .getContainerClient(azureContainerName) + .getProperties(azureKey) + .then( + res => cb(null, res), + err => cb(err), + ), + ], + (err, results) => { + assert.equal(err, null, `Error in assertGetObjects: ${err}`); + const [sourceRes, destRes, azureRes] = results; + const sourceMetadata = normalizeMetadata(sourceRes.Metadata); + const destMetadata = normalizeMetadata(destRes.Metadata); + const convertedMD5 = convertMD5(azureRes[0].contentSettings.contentMD5); + if (objSize && objSize.empty) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(convertedMD5, `${emptyMD5}`); + assert.strictEqual('0', azureRes[0].contentLength); + } else if (objSize && objSize.big) { + assert.strictEqual(sourceRes.ETag, `"${bigMD5}"`); + assert.strictEqual(destRes.ETag, `"${bigMD5}"`); + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { + assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); + assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(convertedMD5, `${bigMD5}`); + } } else { - assert.strictEqual(convertedMD5, `${bigMD5}`); + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { + assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); + assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(sourceRes.ETag, `"${normalMD5}"`); + assert.strictEqual(destRes.ETag, `"${normalMD5}"`); + assert.strictEqual(convertedMD5, `${normalMD5}`); + } } - } else { - if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); - assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); - } else { - assert.strictEqual(sourceRes.ETag, `"${normalMD5}"`); - assert.strictEqual(destRes.ETag, `"${normalMD5}"`); - assert.strictEqual(convertedMD5, `${normalMD5}`); + if (mdDirective === 'COPY') { + assert.strictEqual(sourceMetadata['test-header'], destMetadata['test-header']); + assert.strictEqual(azureRes[0].metadata.test_header, destMetadata['test-header']); } - } - if (mdDirective === 'COPY') { - assert.strictEqual(sourceMetadata['test-header'], - destMetadata['test-header']); - assert.strictEqual(azureRes[0].metadata.test_header, - destMetadata['test-header']); - } - assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); - assert.strictEqual(sourceMetadata[locMetaHeader], sourceLoc); - assert.strictEqual(destMetadata[locMetaHeader], destLoc); - callback(); - }); + assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); + assert.strictEqual(sourceMetadata[locMetaHeader], sourceLoc); + assert.strictEqual(destMetadata[locMetaHeader], destLoc); + callback(); + }, + ); } -describeSkipIfNotMultiple('MultipleBackend object copy: Azure', -function testSuite() { +describeSkipIfNotMultiple('MultipleBackend object copy: Azure', function testSuite() { this.timeout(250000); withV4(sigCfg => { beforeEach(function beFn() { @@ -158,43 +169,48 @@ function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; process.stdout.write('Creating bucket\n'); - s3.createBucketPromise = params => - s3.send(new CreateBucketCommand(params)); + s3.createBucketPromise = params => s3.send(new CreateBucketCommand(params)); if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { s3.createBucketPromise = createEncryptedBucketPromise; } - return s3.createBucketPromise({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: memLocation, - }, - }) - .then(() => s3.createBucketPromise({ Bucket: bucketAzure, - CreateBucketConfiguration: { - LocationConstraint: azureLocation, - }, - })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }); + return s3 + .createBucketPromise({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: memLocation, + }, + }) + .then(() => + s3.createBucketPromise({ + Bucket: bucketAzure, + CreateBucketConfiguration: { + LocationConstraint: azureLocation, + }, + }), + ) + .catch(err => { + process.stdout.write(`Error creating bucket: ${err}\n`); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => bucketUtil.empty(bucketAzure)) - .then(() => { - process.stdout.write(`Deleting bucket: ${bucket}\n`); - return bucketUtil.deleteOne(bucket); - }) - .then(() => { - process.stdout.write(`Deleting bucket: ${bucketAzure}\n`); - return bucketUtil.deleteOne(bucketAzure); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => bucketUtil.empty(bucketAzure)) + .then(() => { + process.stdout.write(`Deleting bucket: ${bucket}\n`); + return bucketUtil.deleteOne(bucket); + }) + .then(() => { + process.stdout.write(`Deleting bucket: ${bucketAzure}\n`); + return bucketUtil.deleteOne(bucketAzure); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should copy an object from mem to Azure', function itFn(done) { @@ -208,19 +224,25 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, memLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + memLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); - it('should copy an object with no location contraint from mem to Azure', - function itFn(done) { + it('should copy an object with no location contraint from mem to Azure', function itFn(done) { putSourceObj(this.test.key, null, null, bucket, () => { const copyParams = { Bucket: bucketAzure, @@ -230,14 +252,21 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, undefined, - this.test.copyKey, bucketAzure, undefined, - this.test.copyKey, 'COPY', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + undefined, + this.test.copyKey, + bucketAzure, + undefined, + this.test.copyKey, + 'COPY', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); @@ -252,14 +281,21 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, memLocation, - this.test.key, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + memLocation, + this.test.key, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); @@ -274,14 +310,21 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, awsLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + awsLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); @@ -296,269 +339,347 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, awsLocation, - this.test.key, 'REPLACE', null, done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + awsLocation, + this.test.key, + 'REPLACE', + null, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); - it('should copy an object from Azure to mem with "REPLACE" directive ' + - 'and no location constraint md', function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, undefined, - this.test.key, 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from Azure to mem with "REPLACE" directive ' + 'and no location constraint md', + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + undefined, + this.test.key, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from mem to Azure with "REPLACE" directive ' + - 'and no location constraint md', function itFn(done) { - putSourceObj(this.test.key, null, null, bucket, () => { - const copyParams = { - Bucket: bucketAzure, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, undefined, - this.test.copyKey, bucketAzure, undefined, - this.test.copyKey, 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from mem to Azure with "REPLACE" directive ' + 'and no location constraint md', + function itFn(done) { + putSourceObj(this.test.key, null, null, bucket, () => { + const copyParams = { + Bucket: bucketAzure, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + undefined, + this.test.copyKey, + bucketAzure, + undefined, + this.test.copyKey, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from Azure to Azure showing sending ' + - 'metadata location constraint this doesn\'t matter with COPY directive', - function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucketAzure, - () => { - const copyParams = { - Bucket: bucketAzure, - Key: this.test.copyKey, - CopySource: `/${bucketAzure}/${this.test.key}`, - MetadataDirective: 'COPY', - Metadata: { 'scal-location-constraint': memLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucketAzure, - azureLocation, - this.test.copyKey, bucketAzure, azureLocation, - this.test.copyKey, 'COPY', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from Azure to Azure showing sending ' + + "metadata location constraint this doesn't matter with COPY directive", + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucketAzure, () => { + const copyParams = { + Bucket: bucketAzure, + Key: this.test.copyKey, + CopySource: `/${bucketAzure}/${this.test.key}`, + MetadataDirective: 'COPY', + Metadata: { 'scal-location-constraint': memLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucketAzure, + azureLocation, + this.test.copyKey, + bucketAzure, + azureLocation, + this.test.copyKey, + 'COPY', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object with no location constraint from Azure to ' + - 'Azure relying on the bucket location constraint', - function itFn(done) { - putSourceObj(this.test.key, null, null, bucketAzure, - () => { - const copyParams = { - Bucket: bucketAzure, - Key: this.test.copyKey, - CopySource: `/${bucketAzure}/${this.test.key}`, - MetadataDirective: 'COPY', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucketAzure, - undefined, this.test.copyKey, bucketAzure, - undefined, this.test.copyKey, 'COPY', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object with no location constraint from Azure to ' + + 'Azure relying on the bucket location constraint', + function itFn(done) { + putSourceObj(this.test.key, null, null, bucketAzure, () => { + const copyParams = { + Bucket: bucketAzure, + Key: this.test.copyKey, + CopySource: `/${bucketAzure}/${this.test.key}`, + MetadataDirective: 'COPY', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucketAzure, + undefined, + this.test.copyKey, + bucketAzure, + undefined, + this.test.copyKey, + 'COPY', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from Azure to mem because bucket ' + - 'destination location is mem', function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'COPY', - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, memLocation, - this.test.key, 'COPY', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from Azure to mem because bucket ' + 'destination location is mem', + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'COPY', + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + memLocation, + this.test.key, + 'COPY', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object on Azure to a different Azure ' + - 'account without source object READ access', - function itFn(done) { - putSourceObj(this.test.key, azureLocation2, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocation2, this.test.copyKey, bucket, - azureLocation, this.test.copyKey, 'REPLACE', null, - done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object on Azure to a different Azure ' + 'account without source object READ access', + function itFn(done) { + putSourceObj(this.test.key, azureLocation2, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation2, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy a 5MB object on Azure to a different Azure ' + - 'account without source object READ access', - function itFn(done) { - putSourceObj(this.test.key, azureLocation2, { big: true }, bucket, - () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${bigMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocation2, this.test.copyKey, bucket, - azureLocation, this.test.copyKey, 'REPLACE', - { big: true }, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy a 5MB object on Azure to a different Azure ' + 'account without source object READ access', + function itFn(done) { + putSourceObj(this.test.key, azureLocation2, { big: true }, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${bigMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation2, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { big: true }, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from bucketmatch=false ' + - 'Azure location to MPU with a bucketmatch=false Azure location', - function itFn(done) { - putSourceObj(this.test.key, azureLocationMismatch, null, bucket, - () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocationMismatch, - this.test.copyKey, bucket, azureLocationMismatch, - `${bucket}/${this.test.copyKey}`, 'REPLACE', null, - done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from bucketmatch=false ' + + 'Azure location to MPU with a bucketmatch=false Azure location', + function itFn(done) { + putSourceObj(this.test.key, azureLocationMismatch, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocationMismatch, + this.test.copyKey, + bucket, + azureLocationMismatch, + `${bucket}/${this.test.copyKey}`, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from bucketmatch=false ' + - 'Azure location to MPU with a bucketmatch=true Azure location', - function itFn(done) { - putSourceObj(this.test.key, azureLocationMismatch, null, bucket, - () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': azureLocation }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocationMismatch, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from bucketmatch=false ' + + 'Azure location to MPU with a bucketmatch=true Azure location', + function itFn(done) { + putSourceObj(this.test.key, azureLocationMismatch, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocation }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocationMismatch, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy an object from bucketmatch=true ' + - 'Azure location to MPU with a bucketmatch=false Azure location', - function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, () => { - const copyParams = { - Bucket: bucket, - Key: this.test.copyKey, - CopySource: `/${bucket}/${this.test.key}`, - MetadataDirective: 'REPLACE', - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - }; - s3.send(new CopyObjectCommand(copyParams)) - .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${normalMD5}"`); - assertGetObjects(this.test.key, bucket, - azureLocation, - this.test.copyKey, bucket, azureLocationMismatch, - `${bucket}/${this.test.copyKey}`, - 'REPLACE', null, done); - }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); - }); - }); + it( + 'should copy an object from bucketmatch=true ' + + 'Azure location to MPU with a bucketmatch=false Azure location', + function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + const copyParams = { + Bucket: bucket, + Key: this.test.copyKey, + CopySource: `/${bucket}/${this.test.key}`, + MetadataDirective: 'REPLACE', + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }; + s3.send(new CopyObjectCommand(copyParams)) + .then(result => { + assert.strictEqual(result.CopyObjectResult.ETag, `"${normalMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + azureLocationMismatch, + `${bucket}/${this.test.copyKey}`, + 'REPLACE', + null, + done, + ); + }) + .catch(err => done(new Error(`Expected success but got error: ${err}`))); + }); + }, + ); - it('should copy a 0-byte object from mem to Azure', - function itFn(done) { - putSourceObj(this.test.key, memLocation, { empty: true }, bucket, - () => { + it('should copy a 0-byte object from mem to Azure', function itFn(done) { + putSourceObj(this.test.key, memLocation, { empty: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -568,21 +689,26 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${emptyMD5}"`); - assertGetObjects(this.test.key, bucket, memLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', { empty: true }, - done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); + assertGetObjects( + this.test.key, + bucket, + memLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { empty: true }, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); it('should copy a 0-byte object on Azure', function itFn(done) { - putSourceObj(this.test.key, azureLocation, { empty: true }, bucket, - () => { + putSourceObj(this.test.key, azureLocation, { empty: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -592,21 +718,26 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${emptyMD5}"`); - assertGetObjects(this.test.key, bucket, azureLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', { empty: true }, - done); + assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { empty: true }, + done, + ); }) - .catch(err => done(new Error( - `Expected success but got error: ${err}`))); + .catch(err => done(new Error(`Expected success but got error: ${err}`))); }); }); it('should copy a 5MB object from mem to Azure', function itFn(done) { - putSourceObj(this.test.key, memLocation, { big: true }, bucket, - () => { + putSourceObj(this.test.key, memLocation, { big: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -616,24 +747,28 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${bigMD5}"`); + assert.strictEqual(result.CopyObjectResult.ETag, `"${bigMD5}"`); setTimeout(() => { - assertGetObjects(this.test.key, bucket, + assertGetObjects( + this.test.key, + bucket, memLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', - { big: true }, done); + this.test.copyKey, + bucket, + azureLocation, + this.test.copyKey, + 'REPLACE', + { big: true }, + done, + ); }, azureTimeout); }) - .catch(err => done(new Error( - `Err copying object: ${err}`))); + .catch(err => done(new Error(`Err copying object: ${err}`))); }); }); it('should copy a 5MB object on Azure', function itFn(done) { - putSourceObj(this.test.key, azureLocation, { big: true }, bucket, - () => { + putSourceObj(this.test.key, azureLocation, { big: true }, bucket, () => { const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -643,29 +778,30 @@ function testSuite() { }; s3.send(new CopyObjectCommand(copyParams)) .then(result => { - assert.strictEqual(result.CopyObjectResult.ETag, - `"${bigMD5}"`); + assert.strictEqual(result.CopyObjectResult.ETag, `"${bigMD5}"`); setTimeout(() => { - assertGetObjects(this.test.key, bucket, + assertGetObjects( + this.test.key, + bucket, + azureLocation, + this.test.copyKey, + bucket, azureLocation, - this.test.copyKey, bucket, azureLocation, - this.test.copyKey, 'REPLACE', - { big: true }, done); + this.test.copyKey, + 'REPLACE', + { big: true }, + done, + ); }, azureTimeout); }) - .catch(err => done(new Error( - `Err copying object: ${err}`))); + .catch(err => done(new Error(`Err copying object: ${err}`))); }); }); - it('should return error if Azure source object has ' + - 'been deleted', function itFn(done) { - putSourceObj(this.test.key, azureLocation, null, bucket, - () => { - azureClient.deleteBlob(azureContainerName, this.test.key, - err => { - assert.equal(err, null, 'Error deleting object from ' + - `Azure: ${err}`); + it('should return error if Azure source object has ' + 'been deleted', function itFn(done) { + putSourceObj(this.test.key, azureLocation, null, bucket, () => { + azureClient.deleteBlob(azureContainerName, this.test.key, err => { + assert.equal(err, null, 'Error deleting object from ' + `Azure: ${err}`); const copyParams = { Bucket: bucket, Key: this.test.copyKey, @@ -673,8 +809,7 @@ function testSuite() { MetadataDirective: 'COPY', }; s3.send(new CopyObjectCommand(copyParams)) - .then(() => done(new Error( - 'Expected ServiceUnavailable error'))) + .then(() => done(new Error('Expected ServiceUnavailable error'))) .catch(err => { assert.strictEqual(err.name, 'ServiceUnavailable'); done(); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js index 4cece36110..d1984b596c 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopy.js @@ -1,6 +1,6 @@ const assert = require('assert'); -const { - S3Client, +const { + S3Client, PutObjectCommand, GetObjectCommand, CopyObjectCommand, @@ -13,11 +13,18 @@ const BucketUtility = require('../../../lib/utility/bucket-util'); const constants = require('../../../../../../constants'); const { config } = require('../../../../../../lib/Config'); const { getRealAwsConfig } = require('../../support/awsConfig'); -const { createEncryptedBucketPromise } = - require('../../../lib/utility/createEncryptedBucket'); -const { describeSkipIfNotMultiple, awsS3, memLocation, awsLocation, - azureLocation, awsLocation2, awsLocationMismatch, awsLocationEncryption, - genUniqID } = require('../utils'); +const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); +const { + describeSkipIfNotMultiple, + awsS3, + memLocation, + awsLocation, + azureLocation, + awsLocation2, + awsLocationMismatch, + awsLocationEncryption, + genUniqID, +} = require('../utils'); const bucket = `objectcopybucket${genUniqID()}`; const bucketAws = `objectcopyaws${genUniqID()}`; @@ -32,7 +39,9 @@ let s3; async function putSourceObj(location, isEmptyObj, bucket) { const key = `somekey-${genUniqID()}`; - const sourceParams = { Bucket: bucket, Key: key, + const sourceParams = { + Bucket: bucket, + Key: key, Metadata: { 'test-header': 'copyme', }, @@ -53,94 +62,105 @@ async function putSourceObj(location, isEmptyObj, bucket) { return key; } -async function assertGetObjects(sourceKey, sourceBucket, sourceLoc, destKey, -destBucket, destLoc, awsKey, mdDirective, isEmptyObj, awsS3, awsLocation) { - const awsBucket = - config.locationConstraints[awsLocation].details.bucketName; +async function assertGetObjects( + sourceKey, + sourceBucket, + sourceLoc, + destKey, + destBucket, + destLoc, + awsKey, + mdDirective, + isEmptyObj, + awsS3, + awsLocation, +) { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey }; const destGetParams = { Bucket: destBucket, Key: destKey }; const awsParams = { Bucket: awsBucket, Key: awsKey }; - + const [sourceRes, destRes, awsRes] = await Promise.all([ s3.send(new GetObjectCommand(sourceGetParams)), s3.send(new GetObjectCommand(destGetParams)), awsS3.send(new GetObjectCommand(awsParams)), ]); - if (isEmptyObj) { - assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); - } else if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); - assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); - } else { - assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); - assert.strictEqual(destRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, destRes.Body); - assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, awsRes.Body); - } - if (destLoc === awsLocationEncryption) { - assert.strictEqual(awsRes.ServerSideEncryption, 'AES256'); - } else { - assert.strictEqual(awsRes.ServerSideEncryption, undefined); - } + if (isEmptyObj) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); + } else if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { + assert.strictEqual(sourceRes.ServerSideEncryption, 'AES256'); + assert.strictEqual(destRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); + assert.strictEqual(destRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, destRes.Body); + assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, awsRes.Body); + } + if (destLoc === awsLocationEncryption) { + assert.strictEqual(awsRes.ServerSideEncryption, 'AES256'); + } else { + assert.strictEqual(awsRes.ServerSideEncryption, undefined); + } + if (mdDirective === 'COPY') { + assert.deepStrictEqual(sourceRes.Metadata['test-header'], destRes.Metadata['test-header']); + } else if (mdDirective === 'REPLACE') { + assert.strictEqual(destRes.Metadata['test-header'], undefined); + } + if (destLoc === awsLocation) { + assert.strictEqual(awsRes.Metadata[locMetaHeader], destLoc); if (mdDirective === 'COPY') { - assert.deepStrictEqual(sourceRes.Metadata['test-header'], - destRes.Metadata['test-header']); + assert.deepStrictEqual(sourceRes.Metadata['test-header'], awsRes.Metadata['test-header']); } else if (mdDirective === 'REPLACE') { - assert.strictEqual(destRes.Metadata['test-header'], - undefined); - } - if (destLoc === awsLocation) { - assert.strictEqual(awsRes.Metadata[locMetaHeader], destLoc); - if (mdDirective === 'COPY') { - assert.deepStrictEqual(sourceRes.Metadata['test-header'], - awsRes.Metadata['test-header']); - } else if (mdDirective === 'REPLACE') { - assert.strictEqual(awsRes.Metadata['test-header'], - undefined); - } + assert.strictEqual(awsRes.Metadata['test-header'], undefined); } + } assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); assert.strictEqual(sourceRes.Metadata[locMetaHeader], sourceLoc); assert.strictEqual(destRes.Metadata[locMetaHeader], destLoc); } -describeSkipIfNotMultiple('MultipleBackend object copy: AWS', -function testSuite() { +describeSkipIfNotMultiple('MultipleBackend object copy: AWS', function testSuite() { this.timeout(250000); withV4(sigCfg => { beforeEach(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; process.stdout.write('Creating bucket\n'); - + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { await createEncryptedBucketPromise({ Bucket: bucket }); await createEncryptedBucketPromise({ Bucket: awsServerSideEncryptionbucket }); await createEncryptedBucketPromise({ Bucket: bucketAws }); } else { - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: memLocation, - }, - })); - - await s3.send(new CreateBucketCommand({ - Bucket: awsServerSideEncryptionbucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocationEncryption, - }, - })); - - await s3.send(new CreateBucketCommand({ - Bucket: bucketAws, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: memLocation, + }, + }), + ); + + await s3.send( + new CreateBucketCommand({ + Bucket: awsServerSideEncryptionbucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocationEncryption, + }, + }), + ); + + await s3.send( + new CreateBucketCommand({ + Bucket: bucketAws, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ); } }); @@ -154,9 +174,7 @@ function testSuite() { await bucketUtil.deleteOne(bucketAws); }); - it('should copy an object from mem to AWS relying on ' + - 'destination bucket location', - async () => { + it('should copy an object from mem to AWS relying on ' + 'destination bucket location', async () => { const key = await putSourceObj(memLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -168,14 +186,22 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, - bucketAws, awsLocation, copyKey, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucketAws, + awsLocation, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from Azure to AWS relying on ' + - 'destination bucket location', - async () => { + it('should copy an object from Azure to AWS relying on ' + 'destination bucket location', async () => { const key = await putSourceObj(azureLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -187,33 +213,53 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, azureLocation, copyKey, - bucketAws, awsLocation, copyKey, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + azureLocation, + copyKey, + bucketAws, + awsLocation, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object without location contraint from mem ' + - 'to AWS relying on destination bucket location', - async () => { - const key = await putSourceObj(null, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucketAws, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'COPY', - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, undefined, copyKey, - bucketAws, undefined, copyKey, 'COPY', false, awsS3, - awsLocation); - }); + it( + 'should copy an object without location contraint from mem ' + + 'to AWS relying on destination bucket location', + async () => { + const key = await putSourceObj(null, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucketAws, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'COPY', + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + undefined, + copyKey, + bucketAws, + undefined, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object from AWS to mem relying on destination ' + - 'bucket location', - async () => { + it('should copy an object from AWS to mem relying on destination ' + 'bucket location', async () => { const key = await putSourceObj(awsLocation, false, bucketAws); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -225,9 +271,19 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucketAws, awsLocation, copyKey, - bucket, memLocation, key, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucketAws, + awsLocation, + copyKey, + bucket, + memLocation, + key, + 'COPY', + false, + awsS3, + awsLocation, + ); }); it('should copy an object from mem to AWS', async () => { @@ -239,18 +295,28 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from mem to AWS with aws server ' + - 'side encryption', async () => { + it('should copy an object from mem to AWS with aws server ' + 'side encryption', async () => { const key = await putSourceObj(memLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -259,37 +325,59 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocationEncryption }, + 'scal-location-constraint': awsLocationEncryption, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, bucket, - awsLocationEncryption, copyKey, 'REPLACE', false, - awsS3, awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucket, + awsLocationEncryption, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from AWS to mem with encryption with ' + - 'REPLACE directive but no location constraint', async () => { - const key = await putSourceObj(awsLocation, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - undefined, key, 'REPLACE', false, - awsS3, awsLocation); - }); + it( + 'should copy an object from AWS to mem with encryption with ' + + 'REPLACE directive but no location constraint', + async () => { + const key = await putSourceObj(awsLocation, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucket, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + undefined, + key, + 'REPLACE', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object on AWS with aws server side ' + - 'encryption', - async () => { + it('should copy an object on AWS with aws server side ' + 'encryption', async () => { const key = await putSourceObj(awsLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -298,18 +386,28 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocationEncryption }, + 'scal-location-constraint': awsLocationEncryption, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocationEncryption, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocationEncryption, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object on AWS with aws server side ' + - 'encrypted bucket', async () => { + it('should copy an object on AWS with aws server side ' + 'encrypted bucket', async () => { const key = await putSourceObj(awsLocation, false, awsServerSideEncryptionbucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -321,33 +419,53 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, awsServerSideEncryptionbucket, - awsLocation, copyKey, awsServerSideEncryptionbucket, - awsLocationEncryption, copyKey, 'COPY', - false, awsS3, awsLocation); + await assertGetObjects( + key, + awsServerSideEncryptionbucket, + awsLocation, + copyKey, + awsServerSideEncryptionbucket, + awsLocationEncryption, + copyKey, + 'COPY', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object from mem to AWS with encryption with ' + - 'REPLACE directive but no location constraint', async () => { - const key = await putSourceObj(null, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucketAws, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, undefined, copyKey, - bucketAws, undefined, copyKey, 'REPLACE', false, - awsS3, awsLocation); - }); + it( + 'should copy an object from mem to AWS with encryption with ' + + 'REPLACE directive but no location constraint', + async () => { + const key = await putSourceObj(null, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucketAws, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + undefined, + copyKey, + bucketAws, + undefined, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object from AWS to mem with "COPY" ' + - 'directive and aws location metadata', - async () => { + it('should copy an object from AWS to mem with "COPY" ' + 'directive and aws location metadata', async () => { const key = await putSourceObj(awsLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; const copyParams = { @@ -356,14 +474,25 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'COPY', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - memLocation, key, 'COPY', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + memLocation, + key, + 'COPY', + false, + awsS3, + awsLocation, + ); }); it('should copy an object on AWS', async () => { @@ -379,88 +508,124 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); - it('should copy an object on AWS location with bucketMatch equals ' + - 'false to a different AWS location with bucketMatch equals true', - async () => { - const key = await putSourceObj(awsLocationMismatch, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - Metadata: { - 'scal-location-constraint': awsLocation }, - }; - process.stdout.write('Copying object\n'); - const result = await s3.send(new CopyObjectCommand(copyParams)); - assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocationMismatch, copyKey, - bucket, awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); - }); + it( + 'should copy an object on AWS location with bucketMatch equals ' + + 'false to a different AWS location with bucketMatch equals true', + async () => { + const key = await putSourceObj(awsLocationMismatch, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucket, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + Metadata: { + 'scal-location-constraint': awsLocation, + }, + }; + process.stdout.write('Copying object\n'); + const result = await s3.send(new CopyObjectCommand(copyParams)); + assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); + await assertGetObjects( + key, + bucket, + awsLocationMismatch, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); + }, + ); - it('should copy an object on AWS to a different AWS location ' + - 'with source object READ access', - async () => { + it('should copy an object on AWS to a different AWS location ' + 'with source object READ access', async () => { const awsConfig2 = getRealAwsConfig(awsLocation2); const awsS3Two = new S3Client(awsConfig2); const copyKey = `copyKey-${genUniqID()}`; - const awsBucket = - config.locationConstraints[awsLocation].details.bucketName; - + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + // giving access to the object on the AWS side const key = await putSourceObj(awsLocation, false, bucket); - await awsS3.send(new PutObjectAclCommand({ - Bucket: awsBucket, - Key: key, - ACL: 'public-read' - })); - + await awsS3.send( + new PutObjectAclCommand({ + Bucket: awsBucket, + Key: key, + ACL: 'public-read', + }), + ); + const copyParams = { Bucket: bucket, Key: copyKey, CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation2 }, + 'scal-location-constraint': awsLocation2, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - - await assertGetObjects(key, bucket, awsLocation, copyKey, - bucket, awsLocation2, copyKey, 'REPLACE', false, - awsS3Two, awsLocation2); - }); - it('should return error AccessDenied copying an object on ' + - 'AWS to a different AWS account without source object READ access', - async () => { - const key = await putSourceObj(awsLocation, false, bucket); - const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, - Key: copyKey, - CopySource: `/${bucket}/${key}`, - MetadataDirective: 'REPLACE', - Metadata: { - 'scal-location-constraint': awsLocation2 }, - }; - process.stdout.write('Copying object\n'); - try { - await s3.send(new CopyObjectCommand(copyParams)); - assert.fail('Expected AccessDenied error'); - } catch (err) { - assert.strictEqual(err.name, 'AccessDenied'); - } + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation2, + copyKey, + 'REPLACE', + false, + awsS3Two, + awsLocation2, + ); }); + it( + 'should return error AccessDenied copying an object on ' + + 'AWS to a different AWS account without source object READ access', + async () => { + const key = await putSourceObj(awsLocation, false, bucket); + const copyKey = `copyKey-${genUniqID()}`; + const copyParams = { + Bucket: bucket, + Key: copyKey, + CopySource: `/${bucket}/${key}`, + MetadataDirective: 'REPLACE', + Metadata: { + 'scal-location-constraint': awsLocation2, + }, + }; + process.stdout.write('Copying object\n'); + try { + await s3.send(new CopyObjectCommand(copyParams)); + assert.fail('Expected AccessDenied error'); + } catch (err) { + assert.strictEqual(err.name, 'AccessDenied'); + } + }, + ); + it('should copy an object on AWS with REPLACE', async () => { const key = await putSourceObj(awsLocation, false, bucket); const copyKey = `copyKey-${genUniqID()}`; @@ -470,14 +635,25 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${correctMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', false, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + false, + awsS3, + awsLocation, + ); }); it('should copy a 0-byte object from mem to AWS', async () => { @@ -489,14 +665,25 @@ function testSuite() { CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', Metadata: { - 'scal-location-constraint': awsLocation }, + 'scal-location-constraint': awsLocation, + }, }; process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); - await assertGetObjects(key, bucket, memLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', true, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + memLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + true, + awsS3, + awsLocation, + ); }); it('should copy a 0-byte object on AWS', async () => { @@ -512,22 +699,30 @@ function testSuite() { process.stdout.write('Copying object\n'); const result = await s3.send(new CopyObjectCommand(copyParams)); assert.strictEqual(result.CopyObjectResult.ETag, `"${emptyMD5}"`); - await assertGetObjects(key, bucket, awsLocation, copyKey, bucket, - awsLocation, copyKey, 'REPLACE', true, awsS3, - awsLocation); + await assertGetObjects( + key, + bucket, + awsLocation, + copyKey, + bucket, + awsLocation, + copyKey, + 'REPLACE', + true, + awsS3, + awsLocation, + ); }); - it('should return error if AWS source object has ' + - 'been deleted', async () => { + it('should return error if AWS source object has ' + 'been deleted', async () => { const key = await putSourceObj(awsLocation, false, bucket); - const awsBucket = - config.locationConstraints[awsLocation].details.bucketName; - + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + await awsS3.send(new DeleteObjectCommand({ Bucket: awsBucket, Key: key })); - + const copyKey = `copyKey-${genUniqID()}`; - const copyParams = { - Bucket: bucket, + const copyParams = { + Bucket: bucket, Key: copyKey, CopySource: `/${bucket}/${key}`, MetadataDirective: 'REPLACE', diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js index 9a07b4d15e..ad5ab19959 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectCopy/objectCopyAwsVersioning.js @@ -46,22 +46,24 @@ function _getCreateBucketParams(bucket, location) { } function createBuckets(testParams, cb) { - const { sourceBucket, sourceLocation, destBucket, destLocation } - = testParams; + const { sourceBucket, sourceLocation, destBucket, destLocation } = testParams; const sourceParams = _getCreateBucketParams(sourceBucket, sourceLocation); const destParams = _getCreateBucketParams(destBucket, destLocation); if (sourceBucket === destBucket) { - return s3.send(new CreateBucketCommand(sourceParams)) + return s3 + .send(new CreateBucketCommand(sourceParams)) .then(() => cb()) .catch(err => cb(err)); } - return async.map([sourceParams, destParams], + return async.map( + [sourceParams, destParams], (createParams, next) => { s3.send(new CreateBucketCommand(createParams)) .then(() => next()) .catch(next); }, - err => cb(err)); + err => cb(err), + ); } function putSourceObj(testParams, cb) { @@ -92,9 +94,16 @@ function putSourceObj(testParams, cb) { } function copyObject(testParams, cb) { - const { sourceBucket, sourceKey, sourceVersionId, sourceVersioningState, - destBucket, directive, destVersioningState, isEmptyObj } - = testParams; + const { + sourceBucket, + sourceKey, + sourceVersionId, + sourceVersioningState, + destBucket, + directive, + destVersioningState, + isEmptyObj, + } = testParams; const destKey = `destkey-${genUniqID()}`; const copyParams = { Bucket: destBucket, @@ -103,11 +112,9 @@ function copyObject(testParams, cb) { MetadataDirective: directive, }; if (sourceVersionId) { - copyParams.CopySource = - `${copyParams.CopySource}?versionId=${sourceVersionId}`; + copyParams.CopySource = `${copyParams.CopySource}?versionId=${sourceVersionId}`; } else if (sourceVersioningState === 'Suspended') { - copyParams.CopySource = - `${copyParams.CopySource}?versionId=null`; + copyParams.CopySource = `${copyParams.CopySource}?versionId=null`; } s3.send(new CopyObjectCommand(copyParams)) .then(data => { @@ -117,26 +124,24 @@ function copyObject(testParams, cb) { assert.strictEqual(data.VersionId, undefined); } const expectedBody = isEmptyObj ? '' : someBody; - return awsGetLatestVerId(destKey, expectedBody, - (err, awsVersionId) => { - if (err) { - cb(err); - return; - } - Object.assign(testParams, { - destKey, - destVersionId: data.VersionId, - awsVersionId, - }); - if (!data.VersionId && destVersioningState === 'Suspended') { - // eslint-disable-next-line no-param-reassign - testParams.destVersionId = 'null'; - } - cb(); + return awsGetLatestVerId(destKey, expectedBody, (err, awsVersionId) => { + if (err) { + cb(err); + return; + } + Object.assign(testParams, { + destKey, + destVersionId: data.VersionId, + awsVersionId, }); + if (!data.VersionId && destVersioningState === 'Suspended') { + // eslint-disable-next-line no-param-reassign + testParams.destVersionId = 'null'; + } + cb(); + }); }) - .catch(err => cb(new Error( - `Error copying object to destination: ${err}`))); + .catch(err => cb(new Error(`Error copying object to destination: ${err}`))); } async function loadBodyBuffer(res) { @@ -169,106 +174,108 @@ function assertGetObjects(testParams, cb) { isEmptyObj, directive, } = testParams; - const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey, - VersionId: sourceVersionId }; - const destGetParams = { Bucket: destBucket, Key: destKey, - VersionId: destVersionId }; - const awsParams = { Bucket: awsBucket, Key: destKey, - VersionId: awsVersionId }; + const sourceGetParams = { Bucket: sourceBucket, Key: sourceKey, VersionId: sourceVersionId }; + const destGetParams = { Bucket: destBucket, Key: destKey, VersionId: destVersionId }; + const awsParams = { Bucket: awsBucket, Key: destKey, VersionId: awsVersionId }; - async.series([ - cb => { - s3.send(new GetObjectCommand(sourceGetParams)) - .then(async res => { - const bodyBuffer = await loadBodyBuffer(res); - if (bodyBuffer !== undefined) { - // eslint-disable-next-line no-param-reassign - res.Body = bodyBuffer; - } - cb(null, res); - }) - .catch(cb); - }, - cb => { - s3.send(new GetObjectCommand(destGetParams)) - .then(async res => { - const bodyBuffer = await loadBodyBuffer(res); - if (bodyBuffer !== undefined) { - // eslint-disable-next-line no-param-reassign - res.Body = bodyBuffer; - } - cb(null, res); - }) - .catch(cb); + async.series( + [ + cb => { + s3.send(new GetObjectCommand(sourceGetParams)) + .then(async res => { + const bodyBuffer = await loadBodyBuffer(res); + if (bodyBuffer !== undefined) { + // eslint-disable-next-line no-param-reassign + res.Body = bodyBuffer; + } + cb(null, res); + }) + .catch(cb); + }, + cb => { + s3.send(new GetObjectCommand(destGetParams)) + .then(async res => { + const bodyBuffer = await loadBodyBuffer(res); + if (bodyBuffer !== undefined) { + // eslint-disable-next-line no-param-reassign + res.Body = bodyBuffer; + } + cb(null, res); + }) + .catch(cb); + }, + cb => awsS3.getObject(awsParams, cb), + ], + (err, results) => { + assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`); + const [sourceRes, destRes, awsRes] = results; + if (isEmptyObj) { + assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); + assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); + } else { + assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); + assert.strictEqual(destRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, destRes.Body); + assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); + assert.deepStrictEqual(sourceRes.Body, awsRes.Body); + } + if (directive === 'COPY') { + assert.deepStrictEqual(sourceRes.Metadata, testMetadata); + assert.deepStrictEqual(sourceRes.Metadata, destRes.Metadata); + assert.deepStrictEqual(sourceRes.Metadata, awsRes.Metadata); + } else if (directive === 'REPLACE') { + assert.deepStrictEqual(destRes.Metadata, {}); + assert.deepStrictEqual(awsRes.Metadata, {}); + } + assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); + cb(); }, - cb => awsS3.getObject(awsParams, cb), - ], (err, results) => { - assert.strictEqual(err, null, `Error in assertGetObjects: ${err}`); - const [sourceRes, destRes, awsRes] = results; - if (isEmptyObj) { - assert.strictEqual(sourceRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(destRes.ETag, `"${emptyMD5}"`); - assert.strictEqual(awsRes.ETag, `"${emptyMD5}"`); - } else { - assert.strictEqual(sourceRes.ETag, `"${correctMD5}"`); - assert.strictEqual(destRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, destRes.Body); - assert.strictEqual(awsRes.ETag, `"${correctMD5}"`); - assert.deepStrictEqual(sourceRes.Body, awsRes.Body); - } - if (directive === 'COPY') { - assert.deepStrictEqual(sourceRes.Metadata, testMetadata); - assert.deepStrictEqual(sourceRes.Metadata, destRes.Metadata); - assert.deepStrictEqual(sourceRes.Metadata, awsRes.Metadata); - } else if (directive === 'REPLACE') { - assert.deepStrictEqual(destRes.Metadata, {}); - assert.deepStrictEqual(awsRes.Metadata, {}); - } - assert.strictEqual(sourceRes.ContentLength, destRes.ContentLength); - cb(); - }); + ); } -describeSkipIfNotMultiple('AWS backend object copy with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend object copy with versioning', function testSuite() { this.timeout(250000); withV4(sigCfg => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - afterEach(() => bucketUtil.empty(sourceBucketName) - .then(() => bucketUtil.deleteOne(sourceBucketName)) - .catch(err => { - process.stdout.write('Error deleting source bucket ' + - `in afterEach: ${err}\n`); - throw err; - }) - .then(() => bucketUtil.empty(destBucketName)) - .then(() => bucketUtil.deleteOne(destBucketName)) - .catch(err => { - if (err.code === 'NoSuchBucket') { - process.stdout.write('Warning: did not find dest bucket ' + - 'for deletion'); - // we do not throw err since dest bucket may not exist - // if we are using source as dest - } else { - process.stdout.write('Error deleting dest bucket ' + - `in afterEach: ${err}\n`); + afterEach(() => + bucketUtil + .empty(sourceBucketName) + .then(() => bucketUtil.deleteOne(sourceBucketName)) + .catch(err => { + process.stdout.write('Error deleting source bucket ' + `in afterEach: ${err}\n`); throw err; - } - }) + }) + .then(() => bucketUtil.empty(destBucketName)) + .then(() => bucketUtil.deleteOne(destBucketName)) + .catch(err => { + if (err.code === 'NoSuchBucket') { + process.stdout.write('Warning: did not find dest bucket ' + 'for deletion'); + // we do not throw err since dest bucket may not exist + // if we are using source as dest + } else { + process.stdout.write('Error deleting dest bucket ' + `in afterEach: ${err}\n`); + throw err; + } + }), ); - [{ - directive: 'REPLACE', - isEmptyObj: true, - }, { - directive: 'REPLACE', - isEmptyObj: false, - }, { - directive: 'COPY', - isEmptyObj: false, - }].forEach(testParams => { + [ + { + directive: 'REPLACE', + isEmptyObj: true, + }, + { + directive: 'REPLACE', + isEmptyObj: false, + }, + { + directive: 'COPY', + isEmptyObj: false, + }, + ].forEach(testParams => { Object.assign(testParams, { sourceBucket: sourceBucketName, sourceLocation: awsLocation, @@ -276,180 +283,215 @@ function testSuite() { destLocation: awsLocation, }); const { isEmptyObj, directive } = testParams; - it(`should copy ${isEmptyObj ? 'an empty' : ''} ` + - 'object from AWS backend non-versioned bucket' + - 'to AWS backend versioned bucket ' + - `with ${directive} directive`, done => { - Object.assign(testParams, { - sourceVersioningState: undefined, - destVersioningState: 'Enabled', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty' : ''} ` + + 'object from AWS backend non-versioned bucket' + + 'to AWS backend versioned bucket ' + + `with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: undefined, + destVersioningState: 'Enabled', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + - 'from one AWS backend versioned bucket' + - `to another on ${directive} directive`, - done => { - Object.assign(testParams, { - sourceVersioningState: 'Enabled', - destVersioningState: 'Enabled', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => enableVersioning(s3, testParams.sourceBucket, next), - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}version ` + + 'from one AWS backend versioned bucket' + + `to another on ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Enabled', + destVersioningState: 'Enabled', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}null ` + - 'version from one AWS backend versioning suspended bucket to ' + - ` another versioning suspended bucket with ${directive} directive`, - done => { - Object.assign(testParams, { - sourceVersioningState: 'Suspended', - destVersioningState: 'Suspended', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => suspendVersioning(s3, testParams.sourceBucket, - next), - next => putSourceObj(testParams, next), - next => suspendVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}null ` + + 'version from one AWS backend versioning suspended bucket to ' + + ` another versioning suspended bucket with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Suspended', + destVersioningState: 'Suspended', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => suspendVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => suspendVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}version ` + - 'from a AWS backend versioned bucket to a versioned-suspended' + - `one with ${directive} directive`, done => { - Object.assign(testParams, { - sourceVersioningState: 'Enabled', - destVersioningState: 'Suspended', - }); - async.waterfall([ - next => createBuckets(testParams, next), - next => enableVersioning(s3, testParams.sourceBucket, next), - next => putSourceObj(testParams, next), - next => suspendVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id from - // copy was stored to get the right version - next => enableVersioning(s3, testParams.destBucket, next), - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}version ` + + 'from a AWS backend versioned bucket to a versioned-suspended' + + `one with ${directive} directive`, + done => { + Object.assign(testParams, { + sourceVersioningState: 'Enabled', + destVersioningState: 'Suspended', + }); + async.waterfall( + [ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + next => putSourceObj(testParams, next), + next => suspendVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id from + // copy was stored to get the right version + next => enableVersioning(s3, testParams.destBucket, next), + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); }); - it('versioning not configured: if copy object to a ' + - 'pre-existing object on AWS backend, metadata should be overwritten ' + - 'but data of previous version in AWS should not be deleted', - function itF(done) { - const destKey = `destkey-${genUniqID()}`; - const testParams = { - sourceBucket: sourceBucketName, - sourceLocation: awsLocation, - sourceVersioningState: undefined, - destBucket: sourceBucketName, - destLocation: awsLocation, - destVersioningState: undefined, - isEmptyObj: true, - directive: 'REPLACE', - }; - async.waterfall([ - next => createBuckets(testParams, next), - next => putToAwsBackend(s3, testParams.destBucket, destKey, - someBody, err => next(err)), - next => awsGetLatestVerId(destKey, someBody, next), - (awsVerId, next) => { - this.test.awsVerId = awsVerId; - next(); - }, - next => putSourceObj(testParams, next), - next => { - s3.send(new CopyObjectCommand({ - Bucket: testParams.destBucket, - Key: destKey, - CopySource: `/${testParams.sourceBucket}` + - `/${testParams.sourceKey}`, - MetadataDirective: testParams.directive, - Metadata: { - 'scal-location-constraint': - testParams.destLocation, + it( + 'versioning not configured: if copy object to a ' + + 'pre-existing object on AWS backend, metadata should be overwritten ' + + 'but data of previous version in AWS should not be deleted', + function itF(done) { + const destKey = `destkey-${genUniqID()}`; + const testParams = { + sourceBucket: sourceBucketName, + sourceLocation: awsLocation, + sourceVersioningState: undefined, + destBucket: sourceBucketName, + destLocation: awsLocation, + destVersioningState: undefined, + isEmptyObj: true, + directive: 'REPLACE', + }; + async.waterfall( + [ + next => createBuckets(testParams, next), + next => putToAwsBackend(s3, testParams.destBucket, destKey, someBody, err => next(err)), + next => awsGetLatestVerId(destKey, someBody, next), + (awsVerId, next) => { + this.test.awsVerId = awsVerId; + next(); }, - })) - .then(res => next(null, res)) - .catch(next); - }, - (copyResult, next) => awsGetLatestVerId(destKey, '', - (err, awsVersionId) => { - testParams.destKey = destKey; - testParams.destVersionId = copyResult.VersionId; - testParams.awsVersionId = awsVersionId; - next(); - }), - next => { - s3.send(new DeleteObjectCommand({ - Bucket: testParams.destBucket, - Key: testParams.destKey, - VersionId: 'null', - })) - .then(res => next(null, res)) - .catch(next); - }, - (delData, next) => getAndAssertResult(s3, { bucket: - testParams.destBucket, key: testParams.destKey, - expectedError: 'NoSuchKey' }, next), - next => awsGetLatestVerId(testParams.destKey, someBody, next), - (awsVerId, next) => { - assert.strictEqual(awsVerId, this.test.awsVerId); - next(); - }, - ], done); - }); + next => putSourceObj(testParams, next), + next => { + s3.send( + new CopyObjectCommand({ + Bucket: testParams.destBucket, + Key: destKey, + CopySource: `/${testParams.sourceBucket}` + `/${testParams.sourceKey}`, + MetadataDirective: testParams.directive, + Metadata: { + 'scal-location-constraint': testParams.destLocation, + }, + }), + ) + .then(res => next(null, res)) + .catch(next); + }, + (copyResult, next) => + awsGetLatestVerId(destKey, '', (err, awsVersionId) => { + testParams.destKey = destKey; + testParams.destVersionId = copyResult.VersionId; + testParams.awsVersionId = awsVersionId; + next(); + }), + next => { + s3.send( + new DeleteObjectCommand({ + Bucket: testParams.destBucket, + Key: testParams.destKey, + VersionId: 'null', + }), + ) + .then(res => next(null, res)) + .catch(next); + }, + (delData, next) => + getAndAssertResult( + s3, + { bucket: testParams.destBucket, key: testParams.destKey, expectedError: 'NoSuchKey' }, + next, + ), + next => awsGetLatestVerId(testParams.destKey, someBody, next), + (awsVerId, next) => { + assert.strictEqual(awsVerId, this.test.awsVerId); + next(); + }, + ], + done, + ); + }, + ); - [{ - sourceLocation: memLocation, - directive: 'REPLACE', - isEmptyObj: true, - }, { - sourceLocation: fileLocation, - directive: 'REPLACE', - isEmptyObj: true, - }, { - sourceLocation: memLocation, - directive: 'COPY', - isEmptyObj: false, - }, { - sourceLocation: fileLocation, - directive: 'COPY', - isEmptyObj: false, - }].forEach(testParams => { + [ + { + sourceLocation: memLocation, + directive: 'REPLACE', + isEmptyObj: true, + }, + { + sourceLocation: fileLocation, + directive: 'REPLACE', + isEmptyObj: true, + }, + { + sourceLocation: memLocation, + directive: 'COPY', + isEmptyObj: false, + }, + { + sourceLocation: fileLocation, + directive: 'COPY', + isEmptyObj: false, + }, + ].forEach(testParams => { Object.assign(testParams, { sourceBucket: sourceBucketName, sourceVersioningState: 'Enabled', @@ -459,36 +501,48 @@ function testSuite() { }); const { sourceLocation, directive, isEmptyObj } = testParams; - it(`should copy ${isEmptyObj ? 'empty ' : ''}object from ` + - `${sourceLocation} to bucket on AWS backend with ` + - `versioning with ${directive}`, done => { - async.waterfall([ - next => createBuckets(testParams, next), - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'empty ' : ''}object from ` + + `${sourceLocation} to bucket on AWS backend with ` + + `versioning with ${directive}`, + done => { + async.waterfall( + [ + next => createBuckets(testParams, next), + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); - it(`should copy ${isEmptyObj ? 'an empty ' : ''}version from ` + - `${sourceLocation} to bucket on AWS backend with ` + - `versioning with ${directive} directive`, done => { - async.waterfall([ - next => createBuckets(testParams, next), - next => enableVersioning(s3, testParams.sourceBucket, next), - // returns a version id which is added to testParams - // to be used in object copy - next => putSourceObj(testParams, next), - next => enableVersioning(s3, testParams.destBucket, next), - next => copyObject(testParams, next), - // put another version to test and make sure version id - // from copy was stored to get the right version - next => putToAwsBackend(s3, destBucketName, - testParams.destKey, wrongVersionBody, () => next()), - next => assertGetObjects(testParams, next), - ], done); - }); + it( + `should copy ${isEmptyObj ? 'an empty ' : ''}version from ` + + `${sourceLocation} to bucket on AWS backend with ` + + `versioning with ${directive} directive`, + done => { + async.waterfall( + [ + next => createBuckets(testParams, next), + next => enableVersioning(s3, testParams.sourceBucket, next), + // returns a version id which is added to testParams + // to be used in object copy + next => putSourceObj(testParams, next), + next => enableVersioning(s3, testParams.destBucket, next), + next => copyObject(testParams, next), + // put another version to test and make sure version id + // from copy was stored to get the right version + next => + putToAwsBackend(s3, destBucketName, testParams.destKey, wrongVersionBody, () => next()), + next => assertGetObjects(testParams, next), + ], + done, + ); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js index b41fb15530..11c17f074f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartAzure.js @@ -16,19 +16,27 @@ const azureMpuUtils = s3middleware.azureHelper.mpuUtils; const { config } = require('../../../../../../lib/Config'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { uniqName, getAzureClient, azureLocation, azureLocationMismatch, - memLocation, awsLocation, awsS3, getOwnerInfo, genUniqID, - describeSkipIfNotMultiple } - = require('../utils'); - +const { + uniqName, + getAzureClient, + azureLocation, + azureLocationMismatch, + memLocation, + awsLocation, + awsS3, + getOwnerInfo, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); let azureContainerName; -if (config.locationConstraints[azureLocation] && -config.locationConstraints[azureLocation].details && -config.locationConstraints[azureLocation].details.azureContainerName) { - azureContainerName = - config.locationConstraints[azureLocation].details.azureContainerName; +if ( + config.locationConstraints[azureLocation] && + config.locationConstraints[azureLocation].details && + config.locationConstraints[azureLocation].details.azureContainerName +) { + azureContainerName = config.locationConstraints[azureLocation].details.azureContainerName; } const memBucketName = `memputcopypartazure${genUniqID()}`; @@ -66,12 +74,8 @@ const result = { MaxParts: 1000, IsTruncated: false, Parts: [], - Initiator: - { ID: ownerID, - DisplayName: ownerDisplayName }, - Owner: - { DisplayName: ownerDisplayName, - ID: ownerID }, + Initiator: { ID: ownerID, DisplayName: ownerDisplayName }, + Owner: { DisplayName: ownerDisplayName, ID: ownerID }, StorageClass: 'STANDARD', }; @@ -79,8 +83,7 @@ let s3; let bucketUtil; function assertCopyPart(infos, cb) { - const { azureContainerName, mpuKeyNameAzure, uploadId, md5, - subPartSize } = infos; + const { azureContainerName, mpuKeyNameAzure, uploadId, md5, subPartSize } = infos; const resultCopy = JSON.parse(JSON.stringify(result)); resultCopy.Bucket = azureContainerName; resultCopy.Key = mpuKeyNameAzure; @@ -89,40 +92,52 @@ function assertCopyPart(infos, cb) { for (let i = 0; i < subPartSize.length; i++) { totalSize = totalSize + subPartSize[i]; } - async.waterfall([ - next => { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: mpuKeyNameAzure, - UploadId: uploadId, - })) - .then(res => { - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${md5}"`, - Size: totalSize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => azureClient.getContainerClient(azureContainerName) - .getBlockBlobClient(mpuKeyNameAzure) - .getBlockList('all').then(res => { - subPartSize.forEach((size, index) => { - const partName = azureMpuUtils.getBlockId(uploadId, 1, index); - assert.strictEqual(res.uncommittedBlocks[index].name, partName); - assert.equal(res.uncommittedBlocks[index].size, size); - }); - next(); - }, err => { - assert.equal(err, null, 'listBlocks: Expected ' + - `success, got error: ${err}`); - next(); - }), - ], cb); + async.waterfall( + [ + next => { + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: mpuKeyNameAzure, + UploadId: uploadId, + }), + ) + .then(res => { + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${md5}"`, + Size: totalSize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + azureClient + .getContainerClient(azureContainerName) + .getBlockBlobClient(mpuKeyNameAzure) + .getBlockList('all') + .then( + res => { + subPartSize.forEach((size, index) => { + const partName = azureMpuUtils.getBlockId(uploadId, 1, index); + assert.strictEqual(res.uncommittedBlocks[index].name, partName); + assert.equal(res.uncommittedBlocks[index].size, size); + }); + next(); + }, + err => { + assert.equal(err, null, 'listBlocks: Expected ' + `success, got error: ${err}`); + next(); + }, + ), + ], + cb, + ); } describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { @@ -135,39 +150,33 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => bucketUtil.empty(memBucketName)) - .then(() => { - process.stdout.write(`Deleting bucket ${azureContainerName}\n`); - return bucketUtil.deleteOne(azureContainerName); - }) - .then(() => { - process.stdout.write(`Deleting bucket ${memBucketName}\n`); - return bucketUtil.deleteOne(memBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => bucketUtil.empty(memBucketName)) + .then(() => { + process.stdout.write(`Deleting bucket ${azureContainerName}\n`); + return bucketUtil.deleteOne(azureContainerName); + }) + .then(() => { + process.stdout.write(`Deleting bucket ${memBucketName}\n`); + return bucketUtil.deleteOne(memBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameNormalAzure = - `normalazure${uniqName(keyObjectAzure)}`; - this.currentTest.keyNameNormalAzureMismatch = - `normalazuremismatch${uniqName(keyObjectAzure)}`; - - this.currentTest.keyNameFiveMbAzure = - `fivembazure${uniqName(keyObjectAzure)}`; - this.currentTest.keyNameFiveMbMem = - `fivembmem${uniqName(keyObjectMemory)}`; - - this.currentTest.mpuKeyNameAzure = - `mpukeyname${uniqName(keyObjectAzure)}`; - this.currentTest.mpuKeyNameMem = - `mpukeyname${uniqName(keyObjectMemory)}`; - this.currentTest.mpuKeyNameAWS = - `mpukeyname${uniqName(keyObjectAWS)}`; + this.currentTest.keyNameNormalAzure = `normalazure${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameNormalAzureMismatch = `normalazuremismatch${uniqName(keyObjectAzure)}`; + + this.currentTest.keyNameFiveMbAzure = `fivembazure${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameFiveMbMem = `fivembmem${uniqName(keyObjectMemory)}`; + + this.currentTest.mpuKeyNameAzure = `mpukeyname${uniqName(keyObjectAzure)}`; + this.currentTest.mpuKeyNameMem = `mpukeyname${uniqName(keyObjectMemory)}`; + this.currentTest.mpuKeyNameAWS = `mpukeyname${uniqName(keyObjectAWS)}`; const paramsAzure = { Bucket: azureContainerName, Key: this.currentTest.mpuKeyNameAzure, @@ -183,86 +192,109 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { Key: this.currentTest.mpuKeyNameAWS, Metadata: { 'scal-location-constraint': awsLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: memBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameNormalAzure, - Body: normalBody, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameNormalAzureMismatch, - Body: normalBody, - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameFiveMbAzure, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameFiveMbMem, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': memLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsAzure)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on Azure: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsMem)) - .then(res => { - this.currentTest.uploadIdMem = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload in memory: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsAWS)) - .then(res => { - this.currentTest.uploadIdAWS = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on AWS: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: memBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameNormalAzure, + Body: normalBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameNormalAzureMismatch, + Body: normalBody, + Metadata: { 'scal-location-constraint': azureLocationMismatch }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameFiveMbAzure, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameFiveMbMem, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': memLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsAzure)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next( + new Error( + `createMultipartUpload on Azure: Expected success, got error: ${err}`, + ), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsMem)) + .then(res => { + this.currentTest.uploadIdMem = res.UploadId; + next(); + }) + .catch(err => + next( + new Error( + `createMultipartUpload in memory: Expected success, got error: ${err}`, + ), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsAWS)) + .then(res => { + this.currentTest.uploadIdAWS = res.UploadId; + next(); + }) + .catch(err => + next( + new Error(`createMultipartUpload on AWS: Expected success, got error: ${err}`), + ), + ); + }, + ], + done, + ); }); afterEach(async function afterEachF() { const paramsAzure = { @@ -285,283 +317,279 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { await s3.send(new AbortMultipartUploadCommand(params)); } }); - it('should copy small part from Azure to MPU with Azure location', - function ifF(done) { + it('should copy small part from Azure to MPU with Azure location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: normalMD5, - subPartSize: [normalBodySize], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: normalMD5, + subPartSize: [normalBodySize], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy small part from Azure location with ' + - 'bucketMatch=false to MPU with Azure location', - function ifF(done) { - const params = { - Bucket: azureContainerName, - CopySource: - `${azureContainerName}/` + - `${this.test.keyNameNormalAzureMismatch}`, - Key: this.test.mpuKeyNameAzure, - PartNumber: 1, - UploadId: this.test.uploadId, - }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: normalMD5, - subPartSize: [normalBodySize], - }; - assertCopyPart(infos, next); - }, - ], done); - }); + it( + 'should copy small part from Azure location with ' + 'bucketMatch=false to MPU with Azure location', + function ifF(done) { + const params = { + Bucket: azureContainerName, + CopySource: `${azureContainerName}/` + `${this.test.keyNameNormalAzureMismatch}`, + Key: this.test.mpuKeyNameAzure, + PartNumber: 1, + UploadId: this.test.uploadId, + }; + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: normalMD5, + subPartSize: [normalBodySize], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); + }, + ); - it('should copy 5 Mb part from Azure to MPU with Azure location', - function ifF(done) { + it('should copy 5 Mb part from Azure to MPU with Azure location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameFiveMbAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameFiveMbAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - subPartSize: [fiveMB], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + subPartSize: [fiveMB], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy part from Azure to MPU with memory location', - function ifF(done) { + it('should copy part from Azure to MPU with memory location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameMem, PartNumber: 1, UploadId: this.test.uploadIdMem, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: memBucketName, - Key: this.test.mpuKeyNameMem, - UploadId: this.test.uploadIdMem, - })) - .then(res => { - const resultCopy = - JSON.parse(JSON.stringify(result)); - resultCopy.Bucket = memBucketName; - resultCopy.Key = this.test.mpuKeyNameMem; - resultCopy.UploadId = this.test.uploadIdMem; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${normalMD5}"`, - Size: normalBodySize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: memBucketName, + Key: this.test.mpuKeyNameMem, + UploadId: this.test.uploadIdMem, + }), + ) + .then(res => { + const resultCopy = JSON.parse(JSON.stringify(result)); + resultCopy.Bucket = memBucketName; + resultCopy.Key = this.test.mpuKeyNameMem; + resultCopy.UploadId = this.test.uploadIdMem; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${normalMD5}"`, + Size: normalBodySize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + ], + done, + ); }); - it('should copy part from Azure to MPU with AWS location', - function ifF(done) { + it('should copy part from Azure to MPU with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAWS, PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.equal(err, null, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${normalMD5}"`); - assert.strictEqual(res.Parts[0].Size, - normalBodySize); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.equal(err, null, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${normalMD5}"`); + assert.strictEqual(res.Parts[0].Size, normalBodySize); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy part from Azure object with range to MPU ' + - 'with AWS location', function ifF(done) { + it('should copy part from Azure object with range to MPU ' + 'with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAWS, CopySourceRange: 'bytes=0-5', PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.equal(err, null, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${sixBytesMD5}"`); - assert.strictEqual(res.Parts[0].Size, 6); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.equal(err, null, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${sixBytesMD5}"`); + assert.strictEqual(res.Parts[0].Size, 6); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy 5 Mb part from a memory location to MPU with ' + - 'Azure location', - function ifF(done) { + it('should copy 5 Mb part from a memory location to MPU with ' + 'Azure location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameFiveMbMem}`, + CopySource: `${azureContainerName}/${this.test.keyNameFiveMbMem}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - subPartSize: [fiveMB], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + subPartSize: [fiveMB], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); describe('with existing part', () => { @@ -577,79 +605,92 @@ describeSkipIfNotMultiple('Put Copy Part to AZURE', function describeF() { .then(() => done()) .catch(done); }); - it('should copy part from Azure to Azure with existing ' + - 'parts', function ifF(done) { + it('should copy part from Azure to Azure with existing ' + 'parts', function ifF(done) { const resultCopy = JSON.parse(JSON.stringify(result)); const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/${this.test.keyNameNormalAzure}`, + CopySource: `${azureContainerName}/${this.test.keyNameNormalAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: azureContainerName, - Key: this.test.mpuKeyNameAzure, - UploadId: this.test.uploadId, - })) - .then(res => { - resultCopy.Bucket = azureContainerName; - resultCopy.Key = this.test.mpuKeyNameAzure; - resultCopy.UploadId = this.test.uploadId; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${oneKbMD5}"`, - Size: oneKb }, - { PartNumber: 2, - LastModified: res.Parts[1].LastModified, - ETag: `"${normalMD5}"`, - Size: 11 }, - ]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => azureClient.getContainerClient(azureContainerName) - .getBlockBlobClient(this.test.mpuKeyNameAzure) - .getBlockList('all').then(res => { - const partName = azureMpuUtils.getBlockId( - this.test.uploadId, 1, 0); - const partName2 = azureMpuUtils.getBlockId( - this.test.uploadId, 2, 0); - assert.strictEqual(res.uncommittedBlocks[0].name, partName); - assert.equal(res.uncommittedBlocks[0].size, oneKb); - assert.strictEqual(res.uncommittedBlocks[1].name, partName2); - assert.equal(res.uncommittedBlocks[1].size, 11); - next(); - }, err => { - assert.equal(err, null, 'listBlocks: Expected ' + - `success, got error: ${err}`); - next(); - }), - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: azureContainerName, + Key: this.test.mpuKeyNameAzure, + UploadId: this.test.uploadId, + }), + ) + .then(res => { + resultCopy.Bucket = azureContainerName; + resultCopy.Key = this.test.mpuKeyNameAzure; + resultCopy.UploadId = this.test.uploadId; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${oneKbMD5}"`, + Size: oneKb, + }, + { + PartNumber: 2, + LastModified: res.Parts[1].LastModified, + ETag: `"${normalMD5}"`, + Size: 11, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + azureClient + .getContainerClient(azureContainerName) + .getBlockBlobClient(this.test.mpuKeyNameAzure) + .getBlockList('all') + .then( + res => { + const partName = azureMpuUtils.getBlockId(this.test.uploadId, 1, 0); + const partName2 = azureMpuUtils.getBlockId(this.test.uploadId, 2, 0); + assert.strictEqual(res.uncommittedBlocks[0].name, partName); + assert.equal(res.uncommittedBlocks[0].size, oneKb); + assert.strictEqual(res.uncommittedBlocks[1].name, partName2); + assert.equal(res.uncommittedBlocks[1].size, 11); + next(); + }, + err => { + assert.equal( + err, + null, + 'listBlocks: Expected ' + `success, got error: ${err}`, + ); + next(); + }, + ), + ], + done, + ); }); }); }); }); }); -describeSkipIfNotMultiple('Put Copy Part to AZURE with large object', -function describeF() { +describeSkipIfNotMultiple('Put Copy Part to AZURE with large object', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -659,54 +700,59 @@ function describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test with large object: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameOneHundredAndFiveMbAzure = - `onehundredandfivembazure${uniqName(keyObjectAzure)}`; - this.currentTest.mpuKeyNameAzure = - `mpukeyname${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameOneHundredAndFiveMbAzure = `onehundredandfivembazure${uniqName(keyObjectAzure)}`; + this.currentTest.mpuKeyNameAzure = `mpukeyname${uniqName(keyObjectAzure)}`; const params = { Bucket: azureContainerName, Key: this.currentTest.mpuKeyNameAzure, Metadata: { 'scal-location-constraint': azureLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyNameOneHundredAndFiveMbAzure, - Body: oneHundredAndFiveMbBody, - Metadata: { 'scal-location-constraint': azureLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyNameOneHundredAndFiveMbAzure, + Body: oneHundredAndFiveMbBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next(new Error(`createMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); afterEach(function afterEachF(done) { const params = { @@ -719,47 +765,43 @@ function describeF() { .catch(done); }); - it('should copy 105 MB part from Azure to MPU with Azure ' + - 'location', function ifF(done) { + it('should copy 105 MB part from Azure to MPU with Azure ' + 'location', function ifF(done) { const params = { Bucket: azureContainerName, - CopySource: - `${azureContainerName}/` + - `${this.test.keyNameOneHundredAndFiveMbAzure}`, + CopySource: `${azureContainerName}/` + `${this.test.keyNameOneHundredAndFiveMbAzure}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, - `"${oneHundredAndFiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - azureContainerName, - mpuKeyNameAzure: - this.test.mpuKeyNameAzure, - uploadId: this.test.uploadId, - md5: oneHundredAndFiveMbMD5, - subPartSize: [100 * 1024 * 1024, 5 * 1024 * 1024], - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${oneHundredAndFiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + azureContainerName, + mpuKeyNameAzure: this.test.mpuKeyNameAzure, + uploadId: this.test.uploadId, + md5: oneHundredAndFiveMbMD5, + subPartSize: [100 * 1024 * 1024, 5 * 1024 * 1024], + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); }); }); }); -describeSkipIfNotMultiple('Put Copy Part to AZURE with complete MPU', -function describeF() { +describeSkipIfNotMultiple('Put Copy Part to AZURE with complete MPU', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -769,138 +811,138 @@ function describeF() { afterEach(() => { process.stdout.write('Emptying bucket azureContainerName\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket azureContainerName\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .then(() => { - process.stdout.write('Emptying bucket awsBucketName\n'); - return bucketUtil.empty(awsBucketName); - }) - .then(() => { - process.stdout.write('Deleting bucket awsBucketName\n'); - return bucketUtil.deleteOne(awsBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket azureContainerName\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .then(() => { + process.stdout.write('Emptying bucket awsBucketName\n'); + return bucketUtil.empty(awsBucketName); + }) + .then(() => { + process.stdout.write('Deleting bucket awsBucketName\n'); + return bucketUtil.deleteOne(awsBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - describe('Basic test with complete MPU from AWS to Azure location: ', - () => { + describe('Basic test with complete MPU from AWS to Azure location: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameAws = - `onehundredandfivembazure${uniqName(keyObjectAWS)}`; - this.currentTest.mpuKeyNameAzure = - `mpukeyname${uniqName(keyObjectAzure)}`; + this.currentTest.keyNameAws = `onehundredandfivembazure${uniqName(keyObjectAWS)}`; + this.currentTest.mpuKeyNameAzure = `mpukeyname${uniqName(keyObjectAzure)}`; const createMpuParams = { Bucket: azureContainerName, Key: this.currentTest.mpuKeyNameAzure, Metadata: { 'scal-location-constraint': azureLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: awsBucketName, - Key: this.currentTest.keyNameAws, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': awsLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(createMpuParams)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: awsBucketName, + Key: this.currentTest.keyNameAws, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(createMpuParams)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next(new Error(`createMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); - it('should copy two 5 MB part from Azure to MPU with Azure ' + - 'location', function ifF(done) { + it('should copy two 5 MB part from Azure to MPU with Azure ' + 'location', function ifF(done) { const uploadParams = { Bucket: azureContainerName, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameAzure, PartNumber: 1, UploadId: this.test.uploadId, }; const uploadParams2 = { Bucket: azureContainerName, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameAzure, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(uploadParams)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new UploadPartCopyCommand(uploadParams2)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const completeMpuParams = { - Bucket: azureContainerName, - Key: this.test.mpuKeyNameAzure, - MultipartUpload: { - Parts: [ - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 1, - }, - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 2, - }, - ], - }, - UploadId: this.test.uploadId, - }; - s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) - .then(res => { - assert.strictEqual(res.Bucket, azureContainerName); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAzure); - next(); - }) - .catch(err => next(new Error( - `completeMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(uploadParams)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send(new UploadPartCopyCommand(uploadParams2)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const completeMpuParams = { + Bucket: azureContainerName, + Key: this.test.mpuKeyNameAzure, + MultipartUpload: { + Parts: [ + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 1, + }, + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 2, + }, + ], + }, + UploadId: this.test.uploadId, + }; + s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) + .then(res => { + assert.strictEqual(res.Bucket, azureContainerName); + assert.strictEqual(res.Key, this.test.mpuKeyNameAzure); + next(); + }) + .catch(err => + next(new Error(`completeMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js index b6e416ab4b..3d99f9a9f4 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectPutCopyPart/objectPutCopyPartGcp.js @@ -15,9 +15,19 @@ const { const { config } = require('../../../../../../lib/Config'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { uniqName, gcpBucketMPU, - gcpClient, gcpLocation, gcpLocationMismatch, memLocation, - awsLocation, awsS3, getOwnerInfo, genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { + uniqName, + gcpBucketMPU, + gcpClient, + gcpLocation, + gcpLocationMismatch, + memLocation, + awsLocation, + awsS3, + getOwnerInfo, + genUniqID, + describeSkipIfNotMultiple, +} = require('../utils'); const bucket = `partcopygcp${genUniqID()}`; @@ -51,12 +61,8 @@ const result = { MaxParts: 1000, IsTruncated: false, Parts: [], - Initiator: - { ID: ownerID, - DisplayName: ownerDisplayName }, - Owner: - { DisplayName: ownerDisplayName, - ID: ownerID }, + Initiator: { ID: ownerID, DisplayName: ownerDisplayName }, + Owner: { DisplayName: ownerDisplayName, ID: ownerID }, StorageClass: 'STANDARD', }; @@ -69,36 +75,46 @@ function assertCopyPart(infos, cb) { resultCopy.Bucket = bucketName; resultCopy.Key = keyName; resultCopy.UploadId = uploadId; - async.waterfall([ - next => { - s3.send(new ListPartsCommand({ - Bucket: bucketName, - Key: keyName, - UploadId: uploadId, - })) - .then(res => { - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${md5}"`, - Size: totalSize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => gcpClient.listParts({ - Bucket: gcpBucketMPU, - Key: keyName, - UploadId: uploadId, - }, (err, res) => { - assert.ifError(err, 'GCP listParts: Expected success,' + - `got error: ${err}`); - assert.strictEqual(res.Contents[0].ETag, `"${md5}"`); - next(); - }), - ], cb); + async.waterfall( + [ + next => { + s3.send( + new ListPartsCommand({ + Bucket: bucketName, + Key: keyName, + UploadId: uploadId, + }), + ) + .then(res => { + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${md5}"`, + Size: totalSize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + gcpClient.listParts( + { + Bucket: gcpBucketMPU, + Key: keyName, + UploadId: uploadId, + }, + (err, res) => { + assert.ifError(err, 'GCP listParts: Expected success,' + `got error: ${err}`); + assert.strictEqual(res.Contents[0].ETag, `"${md5}"`); + next(); + }, + ), + ], + cb, + ); } describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { @@ -107,52 +123,48 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { beforeEach(done => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: gcpLocation, - }, - })) + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }), + ) .then(() => done()) .catch(done); }); afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => bucketUtil.empty(memBucketName)) - .then(() => { - process.stdout.write(`Deleting bucket ${bucket}\n`); - return bucketUtil.deleteOne(bucket); - }) - .then(() => { - process.stdout.write(`Deleting bucket ${memBucketName}\n`); - return bucketUtil.deleteOne(memBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => bucketUtil.empty(memBucketName)) + .then(() => { + process.stdout.write(`Deleting bucket ${bucket}\n`); + return bucketUtil.deleteOne(bucket); + }) + .then(() => { + process.stdout.write(`Deleting bucket ${memBucketName}\n`); + return bucketUtil.deleteOne(memBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('Basic test: ', () => { beforeEach(function beforeFn(done) { - this.currentTest.keyNameNormalGcp = - `normalgcp${uniqName(keyObjectGcp)}`; - this.currentTest.keyNameNormalGcpMismatch = - `normalgcpmismatch${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameNormalGcp = `normalgcp${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameNormalGcpMismatch = `normalgcpmismatch${uniqName(keyObjectGcp)}`; - this.currentTest.keyNameFiveMbGcp = - `fivembgcp${uniqName(keyObjectGcp)}`; - this.currentTest.keyNameFiveMbMem = - `fivembmem${uniqName(keyObjectMemory)}`; + this.currentTest.keyNameFiveMbGcp = `fivembgcp${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameFiveMbMem = `fivembmem${uniqName(keyObjectMemory)}`; - this.currentTest.mpuKeyNameGcp = - `mpukeyname${uniqName(keyObjectGcp)}`; - this.currentTest.mpuKeyNameMem = - `mpukeyname${uniqName(keyObjectMemory)}`; - this.currentTest.mpuKeyNameAWS = - `mpukeyname${uniqName(keyObjectAWS)}`; + this.currentTest.mpuKeyNameGcp = `mpukeyname${uniqName(keyObjectGcp)}`; + this.currentTest.mpuKeyNameMem = `mpukeyname${uniqName(keyObjectMemory)}`; + this.currentTest.mpuKeyNameAWS = `mpukeyname${uniqName(keyObjectAWS)}`; const paramsGcp = { Bucket: bucket, Key: this.currentTest.mpuKeyNameGcp, @@ -168,86 +180,107 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { Key: this.currentTest.mpuKeyNameAWS, Metadata: { 'scal-location-constraint': awsLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: memBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameNormalGcp, - Body: normalBody, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameNormalGcpMismatch, - Body: normalBody, - Metadata: { 'scal-location-constraint': - gcpLocationMismatch }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameFiveMbGcp, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': gcpLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: this.currentTest.keyNameFiveMbMem, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': memLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsGcp)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on gcp: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsMem)) - .then(res => { - this.currentTest.uploadIdMem = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload in memory: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new CreateMultipartUploadCommand(paramsAWS)) - .then(res => { - this.currentTest.uploadIdAWS = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload on AWS: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: memBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameNormalGcp, + Body: normalBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameNormalGcpMismatch, + Body: normalBody, + Metadata: { 'scal-location-constraint': gcpLocationMismatch }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameFiveMbGcp, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': gcpLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: this.currentTest.keyNameFiveMbMem, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': memLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsGcp)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next( + new Error(`createMultipartUpload on gcp: Expected success, got error: ${err}`), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsMem)) + .then(res => { + this.currentTest.uploadIdMem = res.UploadId; + next(); + }) + .catch(err => + next( + new Error( + `createMultipartUpload in memory: Expected success, got error: ${err}`, + ), + ), + ); + }, + next => { + s3.send(new CreateMultipartUploadCommand(paramsAWS)) + .then(res => { + this.currentTest.uploadIdAWS = res.UploadId; + next(); + }) + .catch(err => + next( + new Error(`createMultipartUpload on AWS: Expected success, got error: ${err}`), + ), + ); + }, + ], + done, + ); }); afterEach(function afterFn(done) { @@ -266,301 +299,301 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { Key: this.currentTest.mpuKeyNameAWS, UploadId: this.currentTest.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new AbortMultipartUploadCommand(paramsGcp)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(paramsMem)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new AbortMultipartUploadCommand(paramsAWS)) - .then(() => next()) - .catch(next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new AbortMultipartUploadCommand(paramsGcp)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(paramsMem)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new AbortMultipartUploadCommand(paramsAWS)) + .then(() => next()) + .catch(next); + }, + ], + done, + ); }); - it('should copy small part from GCP to MPU with GCP location', - function itFn(done) { + it('should copy small part from GCP to MPU with GCP location', function itFn(done) { const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: normalMD5, - totalSize: normalBodySize, - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: normalMD5, + totalSize: normalBodySize, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy small part from GCP with bucketMatch=false to ' + - 'MPU with GCP location', - function itFn(done) { - const params = { - Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameNormalGcpMismatch}`, - Key: this.test.mpuKeyNameGcp, - PartNumber: 1, - UploadId: this.test.uploadId, - }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: normalMD5, - totalSize: normalBodySize, - }; - assertCopyPart(infos, next); - }, - ], done); - }); + it( + 'should copy small part from GCP with bucketMatch=false to ' + 'MPU with GCP location', + function itFn(done) { + const params = { + Bucket: bucket, + CopySource: `${bucket}/${this.test.keyNameNormalGcpMismatch}`, + Key: this.test.mpuKeyNameGcp, + PartNumber: 1, + UploadId: this.test.uploadId, + }; + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: normalMD5, + totalSize: normalBodySize, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); + }, + ); - it('should copy 5 Mb part from GCP to MPU with GCP location', - function ifF(done) { + it('should copy 5 Mb part from GCP to MPU with GCP location', function ifF(done) { const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameFiveMbGcp}`, + CopySource: `${bucket}/${this.test.keyNameFiveMbGcp}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - totalSize: fiveMB, - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + totalSize: fiveMB, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); - it('should copy part from GCP to MPU with memory location', - function ifF(done) { + it('should copy part from GCP to MPU with memory location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameMem, PartNumber: 1, UploadId: this.test.uploadIdMem, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: memBucketName, - Key: this.test.mpuKeyNameMem, - UploadId: this.test.uploadIdMem, - })) - .then(res => { - const resultCopy = - JSON.parse(JSON.stringify(result)); - resultCopy.Bucket = memBucketName; - resultCopy.Key = this.test.mpuKeyNameMem; - resultCopy.UploadId = this.test.uploadIdMem; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${normalMD5}"`, - Size: normalBodySize }]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: memBucketName, + Key: this.test.mpuKeyNameMem, + UploadId: this.test.uploadIdMem, + }), + ) + .then(res => { + const resultCopy = JSON.parse(JSON.stringify(result)); + resultCopy.Bucket = memBucketName; + resultCopy.Key = this.test.mpuKeyNameMem; + resultCopy.UploadId = this.test.uploadIdMem; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${normalMD5}"`, + Size: normalBodySize, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + ], + done, + ); }); - it('should copy part from GCP to MPU with AWS location', - function ifF(done) { + it('should copy part from GCP to MPU with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameAWS, PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.ifError(err, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${normalMD5}"`); - assert.strictEqual(res.Parts[0].Size, - normalBodySize); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.ifError(err, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${normalMD5}"`); + assert.strictEqual(res.Parts[0].Size, normalBodySize); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy part from GCP object with range to MPU ' + - 'with AWS location', function ifF(done) { + it('should copy part from GCP object with range to MPU ' + 'with AWS location', function ifF(done) { const params = { Bucket: memBucketName, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameAWS, CopySourceRange: 'bytes=0-5', PartNumber: 1, UploadId: this.test.uploadIdAWS, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const awsBucket = - config.locationConstraints[awsLocation] - .details.bucketName; - awsS3.listParts({ - Bucket: awsBucket, - Key: this.test.mpuKeyNameAWS, - UploadId: this.test.uploadIdAWS, - }, (err, res) => { - assert.ifError(err, - 'listParts: Expected success,' + - ` got error: ${err}`); - assert.strictEqual(res.Bucket, awsBucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameAWS); - assert.strictEqual(res.UploadId, - this.test.uploadIdAWS); - assert.strictEqual(res.Parts.length, 1); - assert.strictEqual(res.Parts[0].PartNumber, 1); - assert.strictEqual(res.Parts[0].ETag, - `"${sixBytesMD5}"`); - assert.strictEqual(res.Parts[0].Size, 6); - next(); - }); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${sixBytesMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const awsBucket = config.locationConstraints[awsLocation].details.bucketName; + awsS3.listParts( + { + Bucket: awsBucket, + Key: this.test.mpuKeyNameAWS, + UploadId: this.test.uploadIdAWS, + }, + (err, res) => { + assert.ifError(err, 'listParts: Expected success,' + ` got error: ${err}`); + assert.strictEqual(res.Bucket, awsBucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameAWS); + assert.strictEqual(res.UploadId, this.test.uploadIdAWS); + assert.strictEqual(res.Parts.length, 1); + assert.strictEqual(res.Parts[0].PartNumber, 1); + assert.strictEqual(res.Parts[0].ETag, `"${sixBytesMD5}"`); + assert.strictEqual(res.Parts[0].Size, 6); + next(); + }, + ); + }, + ], + done, + ); }); - it('should copy 5 Mb part from a memory location to MPU with ' + - 'GCP location', - function ifF(done) { + it('should copy 5 Mb part from a memory location to MPU with ' + 'GCP location', function ifF(done) { const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameFiveMbMem}`, + CopySource: `${bucket}/${this.test.keyNameFiveMbMem}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const infos = { - bucketName: bucket, - keyName: this.test.mpuKeyNameGcp, - uploadId: this.test.uploadId, - md5: fiveMbMD5, - totalSize: fiveMB, - }; - assertCopyPart(infos, next); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const infos = { + bucketName: bucket, + keyName: this.test.mpuKeyNameGcp, + uploadId: this.test.uploadId, + md5: fiveMbMD5, + totalSize: fiveMB, + }; + assertCopyPart(infos, next); + }, + ], + done, + ); }); describe('with existing part', () => { @@ -576,75 +609,82 @@ describeSkipIfNotMultiple('Put Copy Part to GCP', function describeFn() { .then(() => done()) .catch(done); }); - it('should copy part from GCP to GCP with existing ' + - 'parts', function ifF(done) { + it('should copy part from GCP to GCP with existing ' + 'parts', function ifF(done) { const resultCopy = JSON.parse(JSON.stringify(result)); const params = { Bucket: bucket, - CopySource: - `${bucket}/${this.test.keyNameNormalGcp}`, + CopySource: `${bucket}/${this.test.keyNameNormalGcp}`, Key: this.test.mpuKeyNameGcp, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(params)) - .then(res => { - assert.strictEqual(res.ETag, `"${normalMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: this.test.mpuKeyNameGcp, - UploadId: this.test.uploadId, - })) - .then(res => { - resultCopy.Bucket = bucket; - resultCopy.Key = this.test.mpuKeyNameGcp; - resultCopy.UploadId = this.test.uploadId; - resultCopy.Parts = - [{ PartNumber: 1, - LastModified: res.Parts[0].LastModified, - ETag: `"${oneKbMD5}"`, - Size: oneKb }, - { PartNumber: 2, - LastModified: res.Parts[1].LastModified, - ETag: `"${normalMD5}"`, - Size: 11 }, - ]; - assert.deepStrictEqual(res, resultCopy); - next(); - }) - .catch(err => next(new Error( - `listParts: Expected success, got error: ${err}`))); - }, - next => gcpClient.listParts({ - Bucket: gcpBucketMPU, - Key: this.test.mpuKeyNameGcp, - UploadId: this.test.uploadId, - }, (err, res) => { - assert.ifError(err, 'GCP listParts: Expected ' + - `success, got error: ${err}`); - assert.strictEqual( - res.Contents[0].ETag, `"${oneKbMD5}"`); - assert.strictEqual( - res.Contents[1].ETag, `"${normalMD5}"`); - next(); - }), - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(params)) + .then(res => { + assert.strictEqual(res.ETag, `"${normalMD5}"`); + next(); + }) + .catch(err => + next(new Error(`uploadPartCopy: Expected success, got error: ${err}`)), + ); + }, + next => { + s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: this.test.mpuKeyNameGcp, + UploadId: this.test.uploadId, + }), + ) + .then(res => { + resultCopy.Bucket = bucket; + resultCopy.Key = this.test.mpuKeyNameGcp; + resultCopy.UploadId = this.test.uploadId; + resultCopy.Parts = [ + { + PartNumber: 1, + LastModified: res.Parts[0].LastModified, + ETag: `"${oneKbMD5}"`, + Size: oneKb, + }, + { + PartNumber: 2, + LastModified: res.Parts[1].LastModified, + ETag: `"${normalMD5}"`, + Size: 11, + }, + ]; + assert.deepStrictEqual(res, resultCopy); + next(); + }) + .catch(err => next(new Error(`listParts: Expected success, got error: ${err}`))); + }, + next => + gcpClient.listParts( + { + Bucket: gcpBucketMPU, + Key: this.test.mpuKeyNameGcp, + UploadId: this.test.uploadId, + }, + (err, res) => { + assert.ifError(err, 'GCP listParts: Expected ' + `success, got error: ${err}`); + assert.strictEqual(res.Contents[0].ETag, `"${oneKbMD5}"`); + assert.strictEqual(res.Contents[1].ETag, `"${normalMD5}"`); + next(); + }, + ), + ], + done, + ); }); }); }); }); }); -describeSkipIfNotMultiple('Put Copy Part to GCP with complete MPU', -function describeF() { +describeSkipIfNotMultiple('Put Copy Part to GCP with complete MPU', function describeF() { this.timeout(800000); withV4(sigCfg => { beforeEach(() => { @@ -654,138 +694,138 @@ function describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .then(() => { - process.stdout.write('Emptying bucket awsBucketName\n'); - return bucketUtil.empty(awsBucketName); - }) - .then(() => { - process.stdout.write('Deleting bucket awsBucketName\n'); - return bucketUtil.deleteOne(awsBucketName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .then(() => { + process.stdout.write('Emptying bucket awsBucketName\n'); + return bucketUtil.empty(awsBucketName); + }) + .then(() => { + process.stdout.write('Deleting bucket awsBucketName\n'); + return bucketUtil.deleteOne(awsBucketName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - describe('Basic test with complete MPU from AWS to GCP location: ', - () => { + describe('Basic test with complete MPU from AWS to GCP location: ', () => { beforeEach(function beF(done) { - this.currentTest.keyNameAws = - `onehundredandfivembgcp${uniqName(keyObjectAWS)}`; - this.currentTest.mpuKeyNameGcp = - `mpukeyname${uniqName(keyObjectGcp)}`; + this.currentTest.keyNameAws = `onehundredandfivembgcp${uniqName(keyObjectAWS)}`; + this.currentTest.mpuKeyNameGcp = `mpukeyname${uniqName(keyObjectGcp)}`; const createMpuParams = { Bucket: bucket, Key: this.currentTest.mpuKeyNameGcp, Metadata: { 'scal-location-constraint': gcpLocation }, }; - async.waterfall([ - next => { - s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ - Bucket: awsBucketName, - Key: this.currentTest.keyNameAws, - Body: fiveMbBody, - Metadata: { 'scal-location-constraint': awsLocation }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new CreateMultipartUploadCommand(createMpuParams)) - .then(res => { - this.currentTest.uploadId = res.UploadId; - next(); - }) - .catch(err => next(new Error( - `createMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new CreateBucketCommand({ Bucket: awsBucketName })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutObjectCommand({ + Bucket: awsBucketName, + Key: this.currentTest.keyNameAws, + Body: fiveMbBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new CreateMultipartUploadCommand(createMpuParams)) + .then(res => { + this.currentTest.uploadId = res.UploadId; + next(); + }) + .catch(err => + next(new Error(`createMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); - it('should copy two 5 MB part from GCP to MPU with GCP' + - 'location', function ifF(done) { + it('should copy two 5 MB part from GCP to MPU with GCP' + 'location', function ifF(done) { const uploadParams = { Bucket: bucket, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameGcp, PartNumber: 1, UploadId: this.test.uploadId, }; const uploadParams2 = { Bucket: bucket, - CopySource: - `${awsBucketName}/` + - `${this.test.keyNameAws}`, + CopySource: `${awsBucketName}/` + `${this.test.keyNameAws}`, Key: this.test.mpuKeyNameGcp, PartNumber: 2, UploadId: this.test.uploadId, }; - async.waterfall([ - next => { - s3.send(new UploadPartCopyCommand(uploadParams)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - s3.send(new UploadPartCopyCommand(uploadParams2)) - .then(res => { - assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); - next(); - }) - .catch(err => next(new Error( - `uploadPartCopy: Expected success, got error: ${err}`))); - }, - next => { - const completeMpuParams = { - Bucket: bucket, - Key: this.test.mpuKeyNameGcp, - MultipartUpload: { - Parts: [ - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 1, - }, - { - ETag: `"${fiveMbMD5}"`, - PartNumber: 2, - }, - ], - }, - UploadId: this.test.uploadId, - }; - s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) - .then(res => { - assert.strictEqual(res.Bucket, bucket); - assert.strictEqual(res.Key, - this.test.mpuKeyNameGcp); - next(); - }) - .catch(err => next(new Error( - `completeMultipartUpload: Expected success, got error: ${err}`))); - }, - ], done); + async.waterfall( + [ + next => { + s3.send(new UploadPartCopyCommand(uploadParams)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + s3.send(new UploadPartCopyCommand(uploadParams2)) + .then(res => { + assert.strictEqual(res.ETag, `"${fiveMbMD5}"`); + next(); + }) + .catch(err => next(new Error(`uploadPartCopy: Expected success, got error: ${err}`))); + }, + next => { + const completeMpuParams = { + Bucket: bucket, + Key: this.test.mpuKeyNameGcp, + MultipartUpload: { + Parts: [ + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 1, + }, + { + ETag: `"${fiveMbMD5}"`, + PartNumber: 2, + }, + ], + }, + UploadId: this.test.uploadId, + }; + s3.send(new CompleteMultipartUploadCommand(completeMpuParams)) + .then(res => { + assert.strictEqual(res.Bucket, bucket); + assert.strictEqual(res.Key, this.test.mpuKeyNameGcp); + next(); + }) + .catch(err => + next(new Error(`completeMultipartUpload: Expected success, got error: ${err}`)), + ); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js index 976324d4a6..e36f164413 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/objectTagging.js @@ -13,9 +13,19 @@ const { } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { describeSkipIfNotMultiple, awsS3, awsBucket, getAwsRetry, - getAzureClient, getAzureContainerName, convertMD5, memLocation, - fileLocation, awsLocation, azureLocation, genUniqID, +const { + describeSkipIfNotMultiple, + awsS3, + awsBucket, + getAwsRetry, + getAzureClient, + getAzureContainerName, + convertMD5, + memLocation, + fileLocation, + awsLocation, + azureLocation, + genUniqID, } = require('../utils'); const azureClient = getAzureClient(); @@ -52,23 +62,19 @@ const putTags = { const tagObj = { key1: 'value1', key2: 'value2' }; function getAndAssertObjectTags(tagParams, callback) { - return s3.send(new GetObjectTaggingCommand(tagParams)) + return s3 + .send(new GetObjectTaggingCommand(tagParams)) .then(res => { assert.strictEqual(res.TagSet.length, 2); - assert.strictEqual(res.TagSet[0].Key, - putTags.TagSet[0].Key); - assert.strictEqual(res.TagSet[0].Value, - putTags.TagSet[0].Value); - assert.strictEqual(res.TagSet[1].Key, - putTags.TagSet[1].Key); - assert.strictEqual(res.TagSet[1].Value, - putTags.TagSet[1].Value); + assert.strictEqual(res.TagSet[0].Key, putTags.TagSet[0].Key); + assert.strictEqual(res.TagSet[0].Value, putTags.TagSet[0].Value); + assert.strictEqual(res.TagSet[1].Key, putTags.TagSet[1].Key); + assert.strictEqual(res.TagSet[1].Value, putTags.TagSet[1].Value); callback(); }) .catch(callback); } - function awsGet(key, tagCheck, isEmpty, isMpu, callback) { process.stdout.write('Getting object from AWS\n'); getAwsRetry({ key }, 0, (err, res) => { @@ -91,24 +97,29 @@ function awsGet(key, tagCheck, isEmpty, isMpu, callback) { function azureGet(key, tagCheck, isEmpty, callback) { process.stdout.write('Getting object from Azure\n'); - azureClient.getContainerClient(azureContainerName).getProperties(key).then(res => { - const resMD5 = convertMD5(res.contentSettings.contentMD5); - if (isEmpty) { - assert.strictEqual(resMD5, `${emptyMD5}`); - } else { - assert.strictEqual(resMD5, `${correctMD5}`); - } - if (tagCheck) { - assert.strictEqual(res.metadata.tags, - JSON.stringify(tagObj)); - } else { - assert.strictEqual(res.metadata.tags, undefined); - } - return callback(); - }, err => { - assert.equal(err, null); - return callback(); - }); + azureClient + .getContainerClient(azureContainerName) + .getProperties(key) + .then( + res => { + const resMD5 = convertMD5(res.contentSettings.contentMD5); + if (isEmpty) { + assert.strictEqual(resMD5, `${emptyMD5}`); + } else { + assert.strictEqual(resMD5, `${correctMD5}`); + } + if (tagCheck) { + assert.strictEqual(res.metadata.tags, JSON.stringify(tagObj)); + } else { + assert.strictEqual(res.metadata.tags, undefined); + } + return callback(); + }, + err => { + assert.equal(err, null); + return callback(); + }, + ); } function getObject(key, backend, tagCheck, isEmpty, isMpu, callback) { @@ -123,8 +134,7 @@ function getObject(key, backend, tagCheck, isEmpty, isMpu, callback) { } else { assert.strictEqual(res.ETag, `"${correctMD5}"`); } - assert.strictEqual(res.Metadata['scal-location-constraint'], - backend); + assert.strictEqual(res.Metadata['scal-location-constraint'], backend); if (tagCheck) { assert.strictEqual(res.TagCount, 2); } else { @@ -158,37 +168,41 @@ function getObject(key, backend, tagCheck, isEmpty, isMpu, callback) { } function mpuWaterfall(params, cb) { - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand(params)) - .then(data => next(null, data.UploadId)) - .catch(next); - }, - (uploadId, next) => { - const partParams = { Bucket: bucket, Key: params.Key, PartNumber: 1, - UploadId: uploadId, Body: body }; - s3.send(new UploadPartCommand(partParams)) - .then(result => next(null, uploadId, result.ETag)) - .catch(next); - }, - (uploadId, eTag, next) => { - const compParams = { Bucket: bucket, Key: params.Key, - MultipartUpload: { - Parts: [{ ETag: eTag, PartNumber: 1 }], - }, - UploadId: uploadId }; - s3.send(new CompleteMultipartUploadCommand(compParams)) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand(params)) + .then(data => next(null, data.UploadId)) + .catch(next); + }, + (uploadId, next) => { + const partParams = { Bucket: bucket, Key: params.Key, PartNumber: 1, UploadId: uploadId, Body: body }; + s3.send(new UploadPartCommand(partParams)) + .then(result => next(null, uploadId, result.ETag)) + .catch(next); + }, + (uploadId, eTag, next) => { + const compParams = { + Bucket: bucket, + Key: params.Key, + MultipartUpload: { + Parts: [{ ETag: eTag, PartNumber: 1 }], + }, + UploadId: uploadId, + }; + s3.send(new CompleteMultipartUploadCommand(compParams)) + .then(() => next()) + .catch(next); + }, + ], + err => { + assert.equal(err, null); + cb(err); }, - ], err => { - assert.equal(err, null); - cb(err); - }); + ); } -describeSkipIfNotMultiple('Object tagging with multiple backends', -function testSuite() { +describeSkipIfNotMultiple('Object tagging with multiple backends', function testSuite() { if (!process.env.S3_END_TO_END) { this.retries(2); } @@ -197,8 +211,7 @@ function testSuite() { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -206,26 +219,27 @@ function testSuite() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('putObject with tags and putObjectTagging', () => { testBackends.forEach(backend => { const itSkipIfAzure = backend === 'azurebackend' ? it.skip : it; - it(`should put an object with tags to ${backend} backend`, - done => { + it(`should put an object with tags to ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': backend } }, - putParams); + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { @@ -234,8 +248,7 @@ function testSuite() { .catch(done); }); - it(`should put a 0 byte object with tags to ${backend} backend`, - done => { + it(`should put a 0 byte object with tags to ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, @@ -251,16 +264,16 @@ function testSuite() { .catch(done); }); - it(`should put tags to preexisting object in ${backend} ` + - 'backend', done => { + it(`should put tags to preexisting object in ${backend} ` + 'backend', done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Metadata: - { 'scal-location-constraint': backend } }, putParams); + const params = Object.assign( + { Key: key, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; process.stdout.write('Putting object tags\n'); return s3.send(new PutObjectTaggingCommand(putTagParams)); }) @@ -270,8 +283,7 @@ function testSuite() { .catch(done); }); - it('should put tags to preexisting 0 byte object in ' + - `${backend} backend`, done => { + it('should put tags to preexisting 0 byte object in ' + `${backend} backend`, done => { const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, @@ -281,8 +293,7 @@ function testSuite() { process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; process.stdout.write('Putting object tags\n'); return s3.send(new PutObjectTaggingCommand(putTagParams)); }) @@ -292,8 +303,7 @@ function testSuite() { .catch(done); }); - itSkipIfAzure('should put tags to completed MPU ' + - `object in ${backend}`, done => { + itSkipIfAzure('should put tags to completed MPU ' + `object in ${backend}`, done => { const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, @@ -305,8 +315,7 @@ function testSuite() { done(err); return; } - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; process.stdout.write('Putting object\n'); s3.send(new PutObjectTaggingCommand(putTagParams)) .then(() => { @@ -317,44 +326,50 @@ function testSuite() { }); }); - it('should not return error putting tags to correct object ' + - 'version in AWS, even if a delete marker was created directly ' + - 'on AWS before tags are put', - done => { - const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Metadata: - { 'scal-location-constraint': awsLocation } }, putParams); - process.stdout.write('Putting object\n'); - s3.send(new PutObjectCommand(params)) - .then(() => new Promise((resolve, reject) => { - process.stdout.write('Deleting object from AWS\n'); - awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { - if (err) { - reject(err); - return; - } - resolve(); - }); - })) - .then(() => { - const putTagParams = { Bucket: bucket, Key: key, - Tagging: putTags }; - process.stdout.write('Putting object tags\n'); - return s3.send(new PutObjectTaggingCommand(putTagParams)); - }) - .then(() => done()) - .catch(done); - }); + it( + 'should not return error putting tags to correct object ' + + 'version in AWS, even if a delete marker was created directly ' + + 'on AWS before tags are put', + done => { + const key = `somekey-${genUniqID()}`; + const params = Object.assign( + { Key: key, Metadata: { 'scal-location-constraint': awsLocation } }, + putParams, + ); + process.stdout.write('Putting object\n'); + s3.send(new PutObjectCommand(params)) + .then( + () => + new Promise((resolve, reject) => { + process.stdout.write('Deleting object from AWS\n'); + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }), + ) + .then(() => { + const putTagParams = { Bucket: bucket, Key: key, Tagging: putTags }; + process.stdout.write('Putting object tags\n'); + return s3.send(new PutObjectTaggingCommand(putTagParams)); + }) + .then(() => done()) + .catch(done); + }, + ); }); describe('getObjectTagging', () => { testBackends.forEach(backend => { - it(`should get tags from object on ${backend} backend`, - done => { + it(`should get tags from object on ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': backend } }, - putParams); + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { @@ -365,40 +380,47 @@ function testSuite() { }); }); - it('should not return error on getting tags from object that has ' + - 'had a delete marker put directly on AWS', done => { - const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': awsLocation } }, - putParams); - process.stdout.write('Putting object\n'); - s3.send(new PutObjectCommand(params)) - .then(() => new Promise((resolve, reject) => { - process.stdout.write('Deleting object from AWS\n'); - awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { - if (err) { - reject(err); - return; - } - resolve(); - }); - })) - .then(() => { - const tagParams = { Bucket: bucket, Key: key }; - getAndAssertObjectTags(tagParams, done); - }) - .catch(done); - }); + it( + 'should not return error on getting tags from object that has ' + + 'had a delete marker put directly on AWS', + done => { + const key = `somekey-${genUniqID()}`; + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': awsLocation } }, + putParams, + ); + process.stdout.write('Putting object\n'); + s3.send(new PutObjectCommand(params)) + .then( + () => + new Promise((resolve, reject) => { + process.stdout.write('Deleting object from AWS\n'); + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }), + ) + .then(() => { + const tagParams = { Bucket: bucket, Key: key }; + getAndAssertObjectTags(tagParams, done); + }) + .catch(done); + }, + ); }); describe('deleteObjectTagging', () => { testBackends.forEach(backend => { - it(`should delete tags from object on ${backend} backend`, - done => { + it(`should delete tags from object on ${backend} backend`, done => { const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': backend } }, - putParams); + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': backend } }, + putParams, + ); process.stdout.write('Putting object\n'); s3.send(new PutObjectCommand(params)) .then(() => { @@ -412,31 +434,38 @@ function testSuite() { }); }); - it('should not return error on deleting tags from object that ' + - 'has had delete markers put directly on AWS', done => { - const key = `somekey-${genUniqID()}`; - const params = Object.assign({ Key: key, Tagging: tagString, - Metadata: { 'scal-location-constraint': awsLocation } }, - putParams); - process.stdout.write('Putting object\n'); - s3.send(new PutObjectCommand(params)) - .then(() => new Promise((resolve, reject) => { - process.stdout.write('Deleting object from AWS\n'); - awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { - if (err) { - reject(err); - return; - } - resolve(); - }); - })) - .then(() => { - const tagParams = { Bucket: bucket, Key: key }; - return s3.send(new DeleteObjectTaggingCommand(tagParams)); - }) - .then(() => done()) - .catch(done); - }); + it( + 'should not return error on deleting tags from object that ' + + 'has had delete markers put directly on AWS', + done => { + const key = `somekey-${genUniqID()}`; + const params = Object.assign( + { Key: key, Tagging: tagString, Metadata: { 'scal-location-constraint': awsLocation } }, + putParams, + ); + process.stdout.write('Putting object\n'); + s3.send(new PutObjectCommand(params)) + .then( + () => + new Promise((resolve, reject) => { + process.stdout.write('Deleting object from AWS\n'); + awsS3.deleteObject({ Bucket: awsBucket, Key: key }, err => { + if (err) { + reject(err); + return; + } + resolve(); + }); + }), + ) + .then(() => { + const tagParams = { Bucket: bucket, Key: key }; + return s3.send(new DeleteObjectTaggingCommand(tagParams)); + }) + .then(() => done()) + .catch(done); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js index 52a2a91297..812316765d 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-delete.js @@ -15,7 +15,8 @@ const { tagging, genUniqID, } = require('../utils'); -const { PutObjectCommand, +const { + PutObjectCommand, DeleteObjectCommand, CreateBucketCommand, DeleteBucketCommand, @@ -25,213 +26,317 @@ const { putTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging; const bucket = `awsversioningtagdel${genUniqID()}`; const someBody = 'teststring'; -describeSkipIfNotMultiple('AWS backend object delete tagging with versioning ', -function testSuite() { +describeSkipIfNotMultiple('AWS backend object delete tagging with versioning ', function testSuite() { this.timeout(120000); const tags = { key1: 'value1', key2: 'value2' }; withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - beforeEach(done => s3.send(new CreateBucketCommand({ - Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })).then(() => done()).catch(err => done(err))); + beforeEach(done => + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ) + .then(() => done()) + .catch(err => done(err)), + ); afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); - it('versioning not configured: should delete a tag set on the ' + - 'latest version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(data => - next(null, data)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: false }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - expectedVersionId: false }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); - - it('versioning not configured: should delete a tag set on the ' + - 'version if specified (null)', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(data => - next(null, data)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: false }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedVersionId: false }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning not configured: should delete a tag set on the ' + 'latest version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(data => next(null, data)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: false }, next), + (versionId, next) => delTaggingAndAssert(s3, { bucket, key, expectedVersionId: false }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning suspended: should delete a tag set on the latest ' + - 'version if no version is specified', done => { - const data = [undefined, 'test1', 'test2']; + it('versioning not configured: should delete a tag set on the ' + 'version if specified (null)', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, data, next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, expectedVersionId: 'null' }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - expectedVersionId: 'null' }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(data => next(null, data)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: false }, + next, + ), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: 'null', expectedVersionId: false }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning suspended: should delete a tag set on a specific ' + - 'version (null)', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: 'null', expectedVersionId: 'null' }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: 'null' }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning suspended: should delete a tag set on the latest ' + 'version if no version is specified', + done => { + const data = [undefined, 'test1', 'test2']; + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, data, next), + (versionIds, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: 'null' }, next), + (versionId, next) => delTaggingAndAssert(s3, { bucket, key, expectedVersionId: 'null' }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning enabled then suspended: should delete a tag set on ' + - 'a specific (non-null) version if specified', done => { + it('versioning suspended: should delete a tag set on a specific ' + 'version (null)', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(data => - next(null, data)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key, - [someBody], () => next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, () => - next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedVersionId: s3Vid }, - () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], next), + (versionIds, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + next, + ), + (versionId, next) => + delTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: 'null' }, + next, + ), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning enabled: should delete a tag set on the latest ' + - 'version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - expectedVersionId: versionId }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning enabled then suspended: should delete a tag set on ' + + 'a specific (non-null) version if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(data => next(null, data)) + .catch(err => next(err)), + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => + putNullVersionsToAws(s3, bucket, key, [someBody], () => next(null, s3Vid, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + () => next(null, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: s3Vid, expectedVersionId: s3Vid }, () => + next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should delete a tag set on a specific version', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: putData.VersionId, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId, expectedVersionId: versionId }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); - }); + it( + 'versioning enabled: should delete a tag set on the latest ' + 'version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: putData.VersionId }, next), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, expectedVersionId: versionId }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should delete a tag set on a specific ' + - 'version that is not the latest version', done => { + it('versioning enabled: should delete a tag set on a specific version', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - // put another version - (s3Vid, awsVid, next) => s3.send(new PutObjectCommand({ Bucket: bucket, - Key: key, Body: someBody })).then(() => - next(null, s3Vid, awsVid)).catch(err => next(err, s3Vid, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, err => - next(err, s3Vid, awsVid)), - (s3Vid, awsVid, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedVersionId: s3Vid }, - () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: putData.VersionId, expectedVersionId: putData.VersionId }, + next, + ), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, versionId, expectedVersionId: versionId }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning suspended then enabled: should delete a tag set on ' + - 'a specific version (null) if specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - () => next()), - next => awsGetLatestVerId(key, '', next), - (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], - () => next(null, awsVid)), - (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: 'null' }, - () => next(null, awsVid)), - (awsVid, next) => delTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedVersionId: 'null' }, - () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: {} }, next), - ], done); - }); + it( + 'versioning enabled: should delete a tag set on a specific ' + 'version that is not the latest version', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + // put another version + (s3Vid, awsVid, next) => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: someBody })) + .then(() => next(null, s3Vid, awsVid)) + .catch(err => next(err, s3Vid, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + err => next(err, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: s3Vid, expectedVersionId: s3Vid }, () => + next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to delete ' + - 'tags from object that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => awsS3.send(new DeleteObjectCommand({ Bucket: awsBucket, - Key: key, VersionId: awsVid })).then(delData => next(null, delData)).catch(err => next(err)), - (delData, next) => delTaggingAndAssert(s3, { bucket, key, - expectedError: 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'versioning suspended then enabled: should delete a tag set on ' + 'a specific version (null) if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], () => next()), + next => awsGetLatestVerId(key, '', next), + (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], () => next(null, awsVid)), + (awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + () => next(null, awsVid), + ), + (awsVid, next) => + delTaggingAndAssert(s3, { bucket, key, versionId: 'null', expectedVersionId: 'null' }, () => + next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: {} }, next), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to delete ' + - 'tags from object that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })).then(putData => - next(null, putData)).catch(err => next(err)), - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => awsS3.send(new DeleteObjectCommand({ Bucket: awsBucket, - Key: key, VersionId: awsVid })).then(delData => next(null, delData)).catch(err => next(err)), - (delData, next) => delTaggingAndAssert(s3, { bucket, key, - expectedError: 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'should return an ServiceUnavailable if trying to delete ' + + 'tags from object that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => + awsS3 + .send(new DeleteObjectCommand({ Bucket: awsBucket, Key: key, VersionId: awsVid })) + .then(delData => next(null, delData)) + .catch(err => next(err)), + (delData, next) => + delTaggingAndAssert(s3, { bucket, key, expectedError: 'ServiceUnavailable' }, next), + ], + done, + ); + }, + ); + it( + 'should return an ServiceUnavailable if trying to delete ' + + 'tags from object that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand({ Bucket: bucket, Key: key })) + .then(putData => next(null, putData)) + .catch(err => next(err)), + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => + awsS3 + .send(new DeleteObjectCommand({ Bucket: awsBucket, Key: key, VersionId: awsVid })) + .then(delData => next(null, delData)) + .catch(err => next(err)), + (delData, next) => + delTaggingAndAssert(s3, { bucket, key, expectedError: 'ServiceUnavailable' }, next), + ], + done, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js index 2906707b25..ba74cc41c9 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/objectTagging/taggingAwsVersioning-putget.js @@ -23,20 +23,18 @@ const { genUniqID, } = require('../utils'); -const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert, - awsGetAssertTags } = tagging; +const { putTaggingAndAssert, getTaggingAndAssert, delTaggingAndAssert, awsGetAssertTags } = tagging; const bucket = `awsversioningtag${genUniqID()}`; const someBody = 'teststring'; -describeSkipIfNotMultiple('AWS backend object put/get tagging with versioning', -function testSuite() { +describeSkipIfNotMultiple('AWS backend object put/get tagging with versioning', function testSuite() { this.timeout(120000); const tags = { key1: 'value1', key2: 'value2' }; withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - + beforeEach(done => { const command = new CreateBucketCommand({ Bucket: bucket, @@ -54,341 +52,470 @@ function testSuite() { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); - it('versioning not configured: should put/get a tag set on the ' + - 'latest version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: false }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: false }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning not configured: should put/get a tag set on the ' + 'latest version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: false }, next), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: false }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning not configured: should put/get a tag set on a ' + - 'specific version if specified (null)', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: false }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: false }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning not configured: should put/get a tag set on a ' + 'specific version if specified (null)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: false }, + next, + ), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: false }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning suspended: should put/get a tag set on the latest ' + - 'version if no version is specified', done => { - const data = [undefined, 'test1', 'test2']; - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, data, next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, expectedVersionId: 'null' }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: 'null' }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning suspended: should put/get a tag set on the latest ' + 'version if no version is specified', + done => { + const data = [undefined, 'test1', 'test2']; + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, data, next), + (versionIds, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: 'null' }, next), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: 'null' }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning suspended: should put/get a tag set on a specific ' + - 'version (null)', done => { + it('versioning suspended: should put/get a tag set on a specific ' + 'version (null)', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - next), - (versionIds, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: 'null', expectedVersionId: 'null' }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: 'null' }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], next), + (versionIds, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + next, + ), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: 'null' }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); }); - it('versioning enabled then suspended: should put/get a tag set on ' + - 'a specific (non-null) version if specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => putNullVersionsToAws(s3, bucket, key, - [someBody], () => next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, () => - next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedTags: tags, - expectedVersionId: s3Vid }, () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning enabled then suspended: should put/get a tag set on ' + + 'a specific (non-null) version if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => + putNullVersionsToAws(s3, bucket, key, [someBody], () => next(null, s3Vid, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + () => next(null, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: s3Vid, expectedTags: tags, expectedVersionId: s3Vid }, + () => next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should put/get a tag set on the latest ' + - 'version if no version is specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: versionId }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning enabled: should put/get a tag set on the latest ' + 'version if no version is specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: putData.VersionId }, next), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: versionId }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning enabled: should put/get a tag set on a specific version', - done => { + it('versioning enabled: should put/get a tag set on a specific version', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: putData.VersionId, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => getTaggingAndAssert(s3, { bucket, key, - versionId, expectedTags: tags, - expectedVersionId: versionId }, next), - (versionId, next) => awsGetAssertTags({ key, - expectedTags: tags }, next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: putData.VersionId, expectedVersionId: putData.VersionId }, + next, + ), + (versionId, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId, expectedTags: tags, expectedVersionId: versionId }, + next, + ), + (versionId, next) => awsGetAssertTags({ key, expectedTags: tags }, next), + ], + done, + ); }); - it('versioning enabled: should put/get a tag set on a specific version', - done => { + it('versioning enabled: should put/get a tag set on a specific version', done => { const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: putData.VersionId, - expectedVersionId: putData.VersionId }, next), - (versionId, next) => delTaggingAndAssert(s3, { bucket, key, - versionId, expectedVersionId: versionId }, next), - next => awsGetAssertTags({ key, expectedTags: {} }, next), - ], done); + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: putData.VersionId, expectedVersionId: putData.VersionId }, + next, + ), + (versionId, next) => + delTaggingAndAssert(s3, { bucket, key, versionId, expectedVersionId: versionId }, next), + next => awsGetAssertTags({ key, expectedTags: {} }, next), + ], + done, + ); }); - it('versioning enabled: should put/get a tag set on a specific ' + - 'version that is not the latest version', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - // put another version - (s3Vid, awsVid, next) => { - const command = new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: someBody - }); - s3.send(command) - .then(() => next(null, s3Vid, awsVid)) - .catch(err => next(err, s3Vid, awsVid)); - }, - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, err => - next(err, s3Vid, awsVid)), - (s3Vid, awsVid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedTags: tags, - expectedVersionId: s3Vid }, () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning enabled: should put/get a tag set on a specific ' + 'version that is not the latest version', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + // put another version + (s3Vid, awsVid, next) => { + const command = new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: someBody, + }); + s3.send(command) + .then(() => next(null, s3Vid, awsVid)) + .catch(err => next(err, s3Vid, awsVid)); + }, + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + err => next(err, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: s3Vid, expectedTags: tags, expectedVersionId: s3Vid }, + () => next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('versioning suspended then enabled: should put/get a tag set on ' + - 'a specific version (null) if specified', done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => putNullVersionsToAws(s3, bucket, key, [undefined], - () => next()), - next => awsGetLatestVerId(key, '', next), - (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], - () => next(null, awsVid)), - (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: 'null', expectedVersionId: 'null' }, - () => next(null, awsVid)), - (awsVid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: 'null', expectedTags: tags, - expectedVersionId: 'null' }, () => next(null, awsVid)), - (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, - expectedTags: tags }, next), - ], done); - }); + it( + 'versioning suspended then enabled: should put/get a tag set on ' + + 'a specific version (null) if specified', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => putNullVersionsToAws(s3, bucket, key, [undefined], () => next()), + next => awsGetLatestVerId(key, '', next), + (awsVid, next) => putVersionsToAws(s3, bucket, key, [someBody], () => next(null, awsVid)), + (awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: 'null', expectedVersionId: 'null' }, + () => next(null, awsVid), + ), + (awsVid, next) => + getTaggingAndAssert( + s3, + { bucket, key, versionId: 'null', expectedTags: tags, expectedVersionId: 'null' }, + () => next(null, awsVid), + ), + (awsVid, next) => awsGetAssertTags({ key, versionId: awsVid, expectedTags: tags }, next), + ], + done, + ); + }, + ); - it('should get tags for an object even if it was deleted from ' + - 'AWS directly (we rely on s3 metadata)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedVersionId: false }, () => next(null, awsVid)), - (awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (delData, next) => getTaggingAndAssert(s3, { bucket, key, - expectedTags: tags, expectedVersionId: false, - getObject: false }, next), - ], done); - }); + it( + 'should get tags for an object even if it was deleted from ' + 'AWS directly (we rely on s3 metadata)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedVersionId: false }, () => + next(null, awsVid), + ), + (awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (delData, next) => + getTaggingAndAssert( + s3, + { bucket, key, expectedTags: tags, expectedVersionId: false, getObject: false }, + next, + ), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to put ' + - 'tags from object that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', next), - (awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (delData, next) => putTaggingAndAssert(s3, { bucket, key, tags, - expectedError: 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'should return an ServiceUnavailable if trying to put ' + + 'tags from object that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => awsGetLatestVerId(key, '', next), + (awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (delData, next) => + putTaggingAndAssert(s3, { bucket, key, tags, expectedError: 'ServiceUnavailable' }, next), + ], + done, + ); + }, + ); - it('should get tags for an version even if it was deleted from ' + - 'AWS directly (we rely on s3 metadata)', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => enableVersioning(s3, bucket, next), - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => putTaggingAndAssert(s3, { bucket, key, - tags, versionId: s3Vid, expectedVersionId: s3Vid }, - () => next(null, s3Vid, awsVid)), - (s3Vid, awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(() => next(null, s3Vid)) - .catch(err => next(err, s3Vid)); - }, - (s3Vid, next) => getTaggingAndAssert(s3, { bucket, key, - versionId: s3Vid, expectedTags: tags, - expectedVersionId: s3Vid, getObject: false }, next), - ], done); - }); + it( + 'should get tags for an version even if it was deleted from ' + 'AWS directly (we rely on s3 metadata)', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => enableVersioning(s3, bucket, next), + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedVersionId: s3Vid }, + () => next(null, s3Vid, awsVid), + ), + (s3Vid, awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(() => next(null, s3Vid)) + .catch(err => next(err, s3Vid)); + }, + (s3Vid, next) => + getTaggingAndAssert( + s3, + { + bucket, + key, + versionId: s3Vid, + expectedTags: tags, + expectedVersionId: s3Vid, + getObject: false, + }, + next, + ), + ], + done, + ); + }, + ); - it('should return an ServiceUnavailable if trying to put ' + - 'tags on version that was deleted from AWS directly', - done => { - const key = `somekey-${genUniqID()}`; - async.waterfall([ - next => { - const command = new PutObjectCommand({ Bucket: bucket, Key: key }); - s3.send(command) - .then(data => next(null, data)) - .catch(err => next(err)); - }, - (putData, next) => awsGetLatestVerId(key, '', - (err, awsVid) => next(err, putData.VersionId, awsVid)), - (s3Vid, awsVid, next) => { - const command = new DeleteObjectCommand({ - Bucket: awsBucket, - Key: key, - VersionId: awsVid - }); - awsS3.send(command) - .then(() => next(null, s3Vid)) - .catch(err => next(err, s3Vid)); - }, - (s3Vid, next) => putTaggingAndAssert(s3, { bucket, key, tags, - versionId: s3Vid, expectedError: - 'ServiceUnavailable' }, next), - ], done); - }); + it( + 'should return an ServiceUnavailable if trying to put ' + + 'tags on version that was deleted from AWS directly', + done => { + const key = `somekey-${genUniqID()}`; + async.waterfall( + [ + next => { + const command = new PutObjectCommand({ Bucket: bucket, Key: key }); + s3.send(command) + .then(data => next(null, data)) + .catch(err => next(err)); + }, + (putData, next) => + awsGetLatestVerId(key, '', (err, awsVid) => next(err, putData.VersionId, awsVid)), + (s3Vid, awsVid, next) => { + const command = new DeleteObjectCommand({ + Bucket: awsBucket, + Key: key, + VersionId: awsVid, + }); + awsS3 + .send(command) + .then(() => next(null, s3Vid)) + .catch(err => next(err, s3Vid)); + }, + (s3Vid, next) => + putTaggingAndAssert( + s3, + { bucket, key, tags, versionId: s3Vid, expectedError: 'ServiceUnavailable' }, + next, + ), + ], + done, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js index 66a5c81f07..0f198381ac 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/put.js @@ -11,13 +11,18 @@ const { const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { config } = require('../../../../../../lib/Config'); -const { createEncryptedBucketPromise } = - require('../../../lib/utility/createEncryptedBucket'); +const { createEncryptedBucketPromise } = require('../../../lib/utility/createEncryptedBucket'); const { versioningEnabled } = require('../../../lib/utility/versioning-util'); -const { describeSkipIfNotMultiple, getAwsRetry, awsLocation, - awsLocationEncryption, memLocation, fileLocation, genUniqID } - = require('../utils'); +const { + describeSkipIfNotMultiple, + getAwsRetry, + awsLocation, + awsLocationEncryption, + memLocation, + fileLocation, + genUniqID, +} = require('../utils'); const bucket = `putaws${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); @@ -37,7 +42,7 @@ async function getAwsSuccess(key, awsMD5, location) { reject(new Error(`Expected success, got error on direct AWS call: ${err}`)); return; } - + if (location === awsLocationEncryption) { // doesn't check ETag because it's different // with every PUT with encryption @@ -46,8 +51,7 @@ async function getAwsSuccess(key, awsMD5, location) { if (process.env.ENABLE_KMS_ENCRYPTION !== 'true') { assert.strictEqual(res.ETag, `"${awsMD5}"`); } - assert.strictEqual(res.Metadata['scal-location-constraint'], - location); + assert.strictEqual(res.Metadata['scal-location-constraint'], location); resolve(res); }); }); @@ -57,8 +61,7 @@ async function getAwsError(key, expectedError) { return new Promise((resolve, reject) => { getAwsRetry({ key }, 0, err => { try { - assert.notStrictEqual(err, undefined, - 'Expected error but did not find one'); + assert.notStrictEqual(err, undefined, 'Expected error but did not find one'); assert.strictEqual(err.name, expectedError); resolve(); } catch (assertionError) { @@ -71,16 +74,15 @@ async function getAwsError(key, expectedError) { async function awsGetCheck(objectKey, s3MD5, awsMD5, location) { const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: objectKey })); assert.strictEqual(res.ETag, `"${s3MD5}"`); - + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { assert.strictEqual(res.ServerSideEncryption, 'AES256'); } - + process.stdout.write('Getting object from AWS\n'); return await getAwsSuccess(objectKey, awsMD5, location); } - describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { this.timeout(250000); withV4(sigCfg => { @@ -88,7 +90,7 @@ describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; process.stdout.write('Creating bucket\n'); - + if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { await createEncryptedBucketPromise({ Bucket: bucket }); } else { @@ -103,301 +105,386 @@ describeSkipIfNotMultiple('MultipleBackend put object', function testSuite() { }); // aws-sdk now (v2.363.0) returns 'UriParameterError' error - it.skip('should return an error to put request without a valid ' + - 'bucket name', - async () => { - const key = `somekey-${genUniqID()}`; - try { - await s3.send(new PutObjectCommand({ Bucket: '', Key: key })); - throw new Error('Expected failure but got success'); - } catch (err) { - assert.strictEqual(err.code, 'MethodNotAllowed'); - } - }); - - describeSkipIfNotMultiple('with set location from "x-amz-meta-scal-' + - 'location-constraint" header', function describe() { - if (!process.env.S3_END_TO_END) { - this.retries(2); + it.skip('should return an error to put request without a valid ' + 'bucket name', async () => { + const key = `somekey-${genUniqID()}`; + try { + await s3.send(new PutObjectCommand({ Bucket: '', Key: key })); + throw new Error('Expected failure but got success'); + } catch (err) { + assert.strictEqual(err.code, 'MethodNotAllowed'); } + }); - it('should return an error to put request without a valid ' + - 'location constraint', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': 'fail-region' } }; - try { - await s3.send(new PutObjectCommand(params)); - throw new Error('Expected failure but got success'); - } catch (err) { - assert.strictEqual(err.code, 'InvalidArgument'); + describeSkipIfNotMultiple( + 'with set location from "x-amz-meta-scal-' + 'location-constraint" header', + function describe() { + if (!process.env.S3_END_TO_END) { + this.retries(2); } - }); - it('should put an object to mem', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': memLocation }, - }; - - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - }); - - it('should put a 0-byte object to mem', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': memLocation }, - }; - - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${emptyMD5}"`); - }); - - it('should put only metadata to mem with mdonly header', async () => { - const key = `mdonly-${genUniqID()}`; - const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': awsLocation, - 'mdonly': 'true', - 'md5chksum': b64, - 'size': body.length.toString(), - } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - await getAwsError(key, 'NoSuchKey'); - }); - - it('should put actual object with body and mdonly header', async () => { - const key = `mdonly-${genUniqID()}`; - const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); - const params = { Bucket: bucket, Key: key, Body: body, - Metadata: { 'scal-location-constraint': awsLocation, - 'mdonly': 'true', - 'md5chksum': b64, - 'size': body.length.toString(), - } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - await awsGetCheck(key, correctMD5, correctMD5, awsLocation); - }); - - it('should put 0-byte normally with mdonly header', async () => { - const key = `mdonly-${genUniqID()}`; - const b64 = Buffer.from(emptyMD5, 'hex').toString('base64'); - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': awsLocation, - 'mdonly': 'true', - 'md5chksum': b64, - 'size': '0', - } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); - await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); - }); - - it('should put a 0-byte object to AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Metadata: { 'scal-location-constraint': awsLocation }, - }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should return an error to put request without a valid ' + 'location constraint', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': 'fail-region' }, + }; + try { + await s3.send(new PutObjectCommand(params)); + throw new Error('Expected failure but got success'); + } catch (err) { + assert.strictEqual(err.code, 'InvalidArgument'); + } }); - await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); - }); - it('should put an object to file', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation }, - }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put an object to mem', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': memLocation }, + }; + + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - }); - it('should put an object to AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put a 0-byte object to mem', async () => { + const key = `somekey-${genUniqID()}`; + const params = { Bucket: bucket, Key: key, Metadata: { 'scal-location-constraint': memLocation } }; + + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${emptyMD5}"`); }); - await awsGetCheck(key, correctMD5, correctMD5, awsLocation); - }); - - it('should encrypt body only if bucket encrypted putting ' + - 'object to AWS', - async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put only metadata to mem with mdonly header', async () => { + const key = `mdonly-${genUniqID()}`; + const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); + const params = { + Bucket: bucket, + Key: key, + Metadata: { + 'scal-location-constraint': awsLocation, + mdonly: 'true', + md5chksum: b64, + size: body.length.toString(), + }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + await getAwsError(key, 'NoSuchKey'); }); - await getAwsSuccess(key, correctMD5, awsLocation); - }); + it('should put actual object with body and mdonly header', async () => { + const key = `mdonly-${genUniqID()}`; + const b64 = Buffer.from(correctMD5, 'hex').toString('base64'); + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { + 'scal-location-constraint': awsLocation, + mdonly: 'true', + md5chksum: b64, + size: body.length.toString(), + }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation); + }); - it('should put an object to AWS with encryption', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': - awsLocationEncryption } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put 0-byte normally with mdonly header', async () => { + const key = `mdonly-${genUniqID()}`; + const b64 = Buffer.from(emptyMD5, 'hex').toString('base64'); + const params = { + Bucket: bucket, + Key: key, + Metadata: { + 'scal-location-constraint': awsLocation, + mdonly: 'true', + md5chksum: b64, + size: '0', + }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); }); - await awsGetCheck(key, correctMD5, correctMD5, - awsLocationEncryption); - }); - - it('should return a version id putting object to ' + - 'to AWS with versioning enabled', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); - const res = await s3.send(new PutObjectCommand(params)); - assert.strictEqual(res.VersionId); - await getAwsSuccess(key, correctMD5, awsLocation); - }); - - it('should put a large object to AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: bigBody, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put a 0-byte object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { Bucket: bucket, Key: key, Metadata: { 'scal-location-constraint': awsLocation } }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, emptyMD5, emptyMD5, awsLocation); }); - await awsGetCheck(key, bigS3MD5, bigAWSMD5, awsLocation); - }); - it('should put objects with same key to AWS ' + - 'then file, and object should only be present in file', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + it('should put an object to file', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.ETag, `"${correctMD5}"`); }); - params.Metadata = - { 'scal-location-constraint': fileLocation }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put an object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation); }); - const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual( - res.Metadata['scal-location-constraint'], - fileLocation); - return await getAwsError(key, 'NoSuchKey'); - }); - - it('should put objects with same key to file ' + - 'then AWS, and object should only be present on AWS', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should encrypt body only if bucket encrypted putting ' + 'object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await getAwsSuccess(key, correctMD5, awsLocation); }); - params.Metadata = { - 'scal-location-constraint': awsLocation }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put an object to AWS with encryption', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocationEncryption }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocationEncryption); }); - await awsGetCheck(key, correctMD5, correctMD5, - awsLocation); - }); - - it('should put two objects to AWS with same ' + - 'key, and newest object should be returned', async () => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation, - 'unique-header': 'first object' } }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should return a version id putting object to ' + 'to AWS with versioning enabled', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); + const res = await s3.send(new PutObjectCommand(params)); + assert.strictEqual(res.VersionId); + await getAwsSuccess(key, correctMD5, awsLocation); }); - params.Metadata = { 'scal-location-constraint': awsLocation, - 'unique-header': 'second object' }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); + + it('should put a large object to AWS', async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: bigBody, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, bigS3MD5, bigAWSMD5, awsLocation); }); - await awsGetCheck(key, correctMD5, correctMD5, - awsLocation, result => { - assert.strictEqual(result.Metadata - ['unique-header'], 'second object'); - }); - }); - }); + + it( + 'should put objects with same key to AWS ' + 'then file, and object should only be present in file', + async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + params.Metadata = { 'scal-location-constraint': fileLocation }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(res.Metadata['scal-location-constraint'], fileLocation); + return await getAwsError(key, 'NoSuchKey'); + }, + ); + + it( + 'should put objects with same key to file ' + 'then AWS, and object should only be present on AWS', + async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + params.Metadata = { + 'scal-location-constraint': awsLocation, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation); + }, + ); + + it( + 'should put two objects to AWS with same ' + 'key, and newest object should be returned', + async () => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation, 'unique-header': 'first object' }, + }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + params.Metadata = { 'scal-location-constraint': awsLocation, 'unique-header': 'second object' }; + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); + await awsGetCheck(key, correctMD5, correctMD5, awsLocation, result => { + assert.strictEqual(result.Metadata['unique-header'], 'second object'); + }); + }, + ); + }, + ); }); }); -describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', -() => { +describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', () => { withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); @@ -406,71 +493,89 @@ describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', afterEach(async () => { process.stdout.write('Emptying bucket\n'); - await bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + await bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - it('should put an object to mem with no location header', - async () => { + it('should put an object to mem with no location header', async () => { process.stdout.write('Creating bucket\n'); - await s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: memLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: memLocation, + }, + }), + ); process.stdout.write('Putting object\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ETag, `"${correctMD5}"`); }); it('should put an object to file with no location header', async () => { process.stdout.write('Creating bucket\n'); - await s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: fileLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: fileLocation, + }, + }), + ); process.stdout.write('Putting object\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ETag, `"${correctMD5}"`); }); it('should put an object to AWS with no location header', async () => { process.stdout.write('Creating bucket\n'); - await s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: awsLocation, - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: awsLocation, + }, + }), + ); process.stdout.write('Putting object\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - await s3.send(new PutObjectCommand(params)).then(() => { - process.stdout.write('Putting object succeeded\n'); - }).catch(err => { - throw new Error(`Expected success, got error: ${err}`); - }); + await s3 + .send(new PutObjectCommand(params)) + .then(() => { + process.stdout.write('Putting object succeeded\n'); + }) + .catch(err => { + throw new Error(`Expected success, got error: ${err}`); + }); await awsGetCheck(key, correctMD5, correctMD5, undefined); }); }); @@ -484,42 +589,47 @@ describe('MultipleBackend put based on request endpoint', () => { }); after(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in after: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in after: ${err}\n`); + throw err; + }); }); it('should create bucket in corresponding backend', async () => { process.stdout.write('Creating bucket'); - + // Create bucket using AWS SDK v3 await s3.send(new CreateBucketCommand({ Bucket: bucket })); - + const key = `somekey-${genUniqID()}`; - - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body - })); - const locationData = await s3.send(new GetBucketLocationCommand({ Bucket: bucket })); - const objectData = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key - })); + + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); + const locationData = await s3.send(new GetBucketLocationCommand({ Bucket: bucket })); + const objectData = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); const host = s3.config.endpoint.hostname; let endpoint = config.restEndpoints[host]; // s3 returns '' for us-east-1 if (endpoint === 'us-east-1') { endpoint = ''; } - + assert.strictEqual(locationData.LocationConstraint, endpoint); assert.strictEqual(objectData.ETag, `"${correctMD5}"`); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js index 90b0beb2cf..02247fd608 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putAzure.js @@ -1,11 +1,13 @@ const assert = require('assert'); const async = require('async'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, GetObjectCommand, CreateMultipartUploadCommand, AbortMultipartUploadCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); const { @@ -40,7 +42,10 @@ let bucketUtil; let s3; function azureGetCheck(objectKey, azureMD5, azureMetadata, cb) { - azureClient.getContainerClient(azureContainerName).getBlobClient(objectKey).getProperties() + azureClient + .getContainerClient(azureContainerName) + .getBlobClient(objectKey) + .getProperties() .then(res => { const resMD5 = convertMD5(res.contentSettings.contentMD5); assert.strictEqual(resMD5, azureMD5); @@ -50,8 +55,7 @@ function azureGetCheck(objectKey, azureMD5, azureMetadata, cb) { .catch(err => cb(err)); } -describeSkipIfNotMultiple('MultipleBackend put object to AZURE', function -describeF() { +describeSkipIfNotMultiple('MultipleBackend put object to AZURE', function describeF() { this.timeout(250000); withV4(sigCfg => { beforeEach(function beforeEachF() { @@ -62,76 +66,84 @@ describeF() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(azureContainerName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(azureContainerName); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(azureContainerName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(azureContainerName); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); describe('with bucket location header', () => { - beforeEach(done => { - s3.send(new CreateBucketCommand({ - Bucket: azureContainerName, - CreateBucketConfiguration: { - LocationConstraint: azureLocation, - }, - })) - .then(() => done()) - .catch(done); - }); - - it('should return a NotImplemented error if try to put ' + - 'versioning to bucket with Azure location', done => { - const params = { - Bucket: azureContainerName, - VersioningConfiguration: { - Status: 'Enabled', - }, - }; - s3.send(new PutBucketVersioningCommand(params)) - .then(() => { - done(new Error('Expected NotImplemented error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NotImplemented'); - done(); - }); + beforeEach(done => { + s3.send( + new CreateBucketCommand({ + Bucket: azureContainerName, + CreateBucketConfiguration: { + LocationConstraint: azureLocation, + }, + }), + ) + .then(() => done()) + .catch(done); }); - it('should put an object to Azure, with no object location ' + - 'header, based on bucket location', function it(done) { - const params = { - Bucket: azureContainerName, - Key: this.test.keyName, - Body: normalBody, - }; - async.waterfall([ - next => { - s3.send(new PutObjectCommand(params)) - .then(() => setTimeout(() => next(), azureTimeout)) - .catch(next); - }, - next => azureGetCheck(this.test.keyName, normalMD5, {}, - next), - ], done); - }); + it( + 'should return a NotImplemented error if try to put ' + 'versioning to bucket with Azure location', + done => { + const params = { + Bucket: azureContainerName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }; + s3.send(new PutBucketVersioningCommand(params)) + .then(() => { + done(new Error('Expected NotImplemented error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotImplemented'); + done(); + }); + }, + ); + + it( + 'should put an object to Azure, with no object location ' + 'header, based on bucket location', + function it(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Body: normalBody, + }; + async.waterfall( + [ + next => { + s3.send(new PutObjectCommand(params)) + .then(() => setTimeout(() => next(), azureTimeout)) + .catch(next); + }, + next => azureGetCheck(this.test.keyName, normalMD5, {}, next), + ], + done, + ); + }, + ); }); describe('with no bucket location header', () => { beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: azureContainerName })) - .catch(err => { + s3.send(new CreateBucketCommand({ Bucket: azureContainerName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; - })); + }), + ); keys.forEach(key => { - it(`should put a ${key.describe} object to Azure`, - function itF(done) { + it(`should put a ${key.describe} object to Azure`, function itF(done) { const params = { Bucket: azureContainerName, Key: this.test.keyName, @@ -140,22 +152,20 @@ describeF() { }; s3.send(new PutObjectCommand(params)) .then(() => { - setTimeout(() => - azureGetCheck(this.test.keyName, - key.MD5, azureMetadata, - () => done()), azureTimeout); + setTimeout( + () => azureGetCheck(this.test.keyName, key.MD5, azureMetadata, () => done()), + azureTimeout, + ); }) .catch(done); }); }); - it('should put a object to Azure location with bucketMatch=false', - function itF(done) { + it('should put a object to Azure location with bucketMatch=false', function itF(done) { const params = { Bucket: azureContainerName, Key: this.test.keyName, - Metadata: { 'scal-location-constraint': - azureLocationMismatch }, + Metadata: { 'scal-location-constraint': azureLocationMismatch }, Body: normalBody, }; const azureMetadataMismatch = { @@ -165,17 +175,21 @@ describeF() { }; s3.send(new PutObjectCommand(params)) .then(() => { - setTimeout(() => - azureGetCheck( - `${azureContainerName}/${this.test.keyName}`, - normalMD5, azureMetadataMismatch, - () => done()), azureTimeout); + setTimeout( + () => + azureGetCheck( + `${azureContainerName}/${this.test.keyName}`, + normalMD5, + azureMetadataMismatch, + () => done(), + ), + azureTimeout, + ); }) .catch(done); }); - it('should return error ServiceUnavailable putting an invalid ' + - 'key name to Azure', done => { + it('should return error ServiceUnavailable putting an invalid ' + 'key name to Azure', done => { const params = { Bucket: azureContainerName, Key: '.', @@ -192,18 +206,20 @@ describeF() { }); }); - it('should return error NotImplemented putting a ' + - 'version to Azure', function itF(done) { - s3.send(new PutBucketVersioningCommand({ - Bucket: azureContainerName, - VersioningConfiguration: versioningEnabled, - })) + it('should return error NotImplemented putting a ' + 'version to Azure', function itF(done) { + s3.send( + new PutBucketVersioningCommand({ + Bucket: azureContainerName, + VersioningConfiguration: versioningEnabled, + }), + ) .then(() => { - const params = { Bucket: azureContainerName, + const params = { + Bucket: azureContainerName, Key: this.test.keyName, Body: normalBody, - Metadata: { 'scal-location-constraint': - azureLocation } }; + Metadata: { 'scal-location-constraint': azureLocation }, + }; return s3.send(new PutObjectCommand(params)); }) .then(() => { @@ -215,110 +231,128 @@ describeF() { }); }); - it('should put two objects to Azure with same ' + - 'key, and newest object should be returned', function itF(done) { - const params = { - Bucket: azureContainerName, - Key: this.test.keyName, - Metadata: { 'scal-location-constraint': azureLocation }, - }; - async.waterfall([ - next => { - s3.send(new PutObjectCommand(params)) - .then(() => next()) - .catch(next); - }, - next => { - params.Body = normalBody; - s3.send(new PutObjectCommand(params)) - .then(() => setTimeout(() => next(), azureTimeout)) - .catch(next); - }, - next => { - setTimeout(() => { - azureGetCheck(this.test.keyName, normalMD5, - azureMetadata, next); - }, azureTimeout); - }, - ], done); - }); + it( + 'should put two objects to Azure with same ' + 'key, and newest object should be returned', + function itF(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Metadata: { 'scal-location-constraint': azureLocation }, + }; + async.waterfall( + [ + next => { + s3.send(new PutObjectCommand(params)) + .then(() => next()) + .catch(next); + }, + next => { + params.Body = normalBody; + s3.send(new PutObjectCommand(params)) + .then(() => setTimeout(() => next(), azureTimeout)) + .catch(next); + }, + next => { + setTimeout(() => { + azureGetCheck(this.test.keyName, normalMD5, azureMetadata, next); + }, azureTimeout); + }, + ], + done, + ); + }, + ); - it('should put objects with same key to Azure ' + - 'then file, and object should only be present in file', function - itF(done) { - const params = { - Bucket: azureContainerName, - Key: this.test.keyName, - Body: normalBody, - Metadata: { 'scal-location-constraint': azureLocation } }; - async.waterfall([ - next => { - s3.send(new PutObjectCommand(params)) - .then(() => next()) - .catch(next); - }, - next => { - params.Metadata = { 'scal-location-constraint': - fileLocation }; - s3.send(new PutObjectCommand(params)) - .then(() => setTimeout(() => next(), azureTimeout)) - .catch(next); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: azureContainerName, - Key: this.test.keyName, - })) - .then(res => { - assert.strictEqual( - res.Metadata['scal-location-constraint'], - fileLocation); - next(); - }) - .catch(next); - }, - next => { - azureClient.getContainerClient(azureContainerName) - .getBlobClient(this.test.keyName).getProperties() - .then(() => { - next(new Error('Expected NotFound error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - next(); - }); - }, - ], done); - }); + it( + 'should put objects with same key to Azure ' + 'then file, and object should only be present in file', + function itF(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Body: normalBody, + Metadata: { 'scal-location-constraint': azureLocation }, + }; + async.waterfall( + [ + next => { + s3.send(new PutObjectCommand(params)) + .then(() => next()) + .catch(next); + }, + next => { + params.Metadata = { 'scal-location-constraint': fileLocation }; + s3.send(new PutObjectCommand(params)) + .then(() => setTimeout(() => next(), azureTimeout)) + .catch(next); + }, + next => { + s3.send( + new GetObjectCommand({ + Bucket: azureContainerName, + Key: this.test.keyName, + }), + ) + .then(res => { + assert.strictEqual(res.Metadata['scal-location-constraint'], fileLocation); + next(); + }) + .catch(next); + }, + next => { + azureClient + .getContainerClient(azureContainerName) + .getBlobClient(this.test.keyName) + .getProperties() + .then(() => { + next(new Error('Expected NotFound error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotFound'); + next(); + }); + }, + ], + done, + ); + }, + ); - it('should put objects with same key to file ' + - 'then Azure, and object should only be present on Azure', - function itF(done) { - const params = { Bucket: azureContainerName, Key: - this.test.keyName, - Body: normalBody, - Metadata: { 'scal-location-constraint': fileLocation } }; - async.waterfall([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => { - params.Metadata = { - 'scal-location-constraint': azureLocation, - }; - s3.send(new PutObjectCommand(params)).then(() => setTimeout(() => - next(), azureTimeout)); - }, - next => azureGetCheck(this.test.keyName, normalMD5, - azureMetadata, next), - ], done); - }); + it( + 'should put objects with same key to file ' + 'then Azure, and object should only be present on Azure', + function itF(done) { + const params = { + Bucket: azureContainerName, + Key: this.test.keyName, + Body: normalBody, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + async.waterfall( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => { + params.Metadata = { + 'scal-location-constraint': azureLocation, + }; + s3.send(new PutObjectCommand(params)).then(() => + setTimeout(() => next(), azureTimeout), + ); + }, + next => azureGetCheck(this.test.keyName, normalMD5, azureMetadata, next), + ], + done, + ); + }, + ); describe('with ongoing MPU with same key name', () => { beforeEach(function beFn(done) { - s3.send(new CreateMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyName, - Metadata: { 'scal-location-constraint': azureLocation }, - })) + s3.send( + new CreateMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyName, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) .then(res => { this.currentTest.uploadId = res.UploadId; done(); @@ -327,11 +361,13 @@ describeF() { }); afterEach(function afFn(done) { - s3.send(new AbortMultipartUploadCommand({ - Bucket: azureContainerName, - Key: this.currentTest.keyName, - UploadId: this.currentTest.uploadId, - })) + s3.send( + new AbortMultipartUploadCommand({ + Bucket: azureContainerName, + Key: this.currentTest.keyName, + UploadId: this.currentTest.uploadId, + }), + ) .then(() => { done(); }) @@ -339,11 +375,13 @@ describeF() { }); it('should return ServiceUnavailable', function itFn(done) { - s3.send(new PutObjectCommand({ - Bucket: azureContainerName, - Key: this.test.keyName, - Metadata: { 'scal-location-constraint': azureLocation }, - })) + s3.send( + new PutObjectCommand({ + Bucket: azureContainerName, + Key: this.test.keyName, + Metadata: { 'scal-location-constraint': azureLocation }, + }), + ) .then(() => { done(new Error('Expected ServiceUnavailable error')); }) diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js index 922b28399e..bfc818cce6 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/put/putGcp.js @@ -1,14 +1,9 @@ const assert = require('assert'); -const { - CreateBucketCommand, - PutObjectCommand, - GetObjectCommand, -} = require('@aws-sdk/client-s3'); +const { CreateBucketCommand, PutObjectCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../../support/withV4'); const BucketUtility = require('../../../lib/utility/bucket-util'); -const { gcpClient, gcpBucket, - gcpLocation, fileLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); +const { gcpClient, gcpBucket, gcpLocation, fileLocation, genUniqID, describeSkipIfNotMultiple } = require('../utils'); const bucket = `putgcp${genUniqID()}`; const body = Buffer.from('I am a body', 'utf8'); @@ -25,35 +20,37 @@ const retryTimeout = 10000; const maxGetRetries = 3; function checkGcp(key, gcpMD5, location, callback) { - gcpClient.getObject({ - Bucket: gcpBucket, - Key: key, - }, (err, res) => { - assert.equal(err, null, `Expected success, got error ${err}`); - if (res.Metadata && res.Metadata['scal-etag']) { - assert.strictEqual(res.Metadata['scal-etag'], gcpMD5); - } else { - assert.strictEqual( - res.ETag.substring(1, res.ETag.length - 1), gcpMD5); - } - assert.strictEqual(res.Metadata['scal-location-constraint'], - location); - callback(res); - }); + gcpClient.getObject( + { + Bucket: gcpBucket, + Key: key, + }, + (err, res) => { + assert.equal(err, null, `Expected success, got error ${err}`); + if (res.Metadata && res.Metadata['scal-etag']) { + assert.strictEqual(res.Metadata['scal-etag'], gcpMD5); + } else { + assert.strictEqual(res.ETag.substring(1, res.ETag.length - 1), gcpMD5); + } + assert.strictEqual(res.Metadata['scal-location-constraint'], location); + callback(res); + }, + ); } function checkGcpError(key, expectedError, callback) { setTimeout(() => { - gcpClient.getObject({ - Bucket: gcpBucket, - Key: key, - }, err => { - assert.notStrictEqual(err, undefined, - 'Expected error but did not find one'); - assert.strictEqual(err.code, expectedError, - `Expected error code ${expectedError} but got ${err.code}`); - callback(); - }); + gcpClient.getObject( + { + Bucket: gcpBucket, + Key: key, + }, + err => { + assert.notStrictEqual(err, undefined, 'Expected error but did not find one'); + assert.strictEqual(err.code, expectedError, `Expected error code ${expectedError} but got ${err.code}`); + callback(); + }, + ); }, 1000); } @@ -65,8 +62,7 @@ function gcpGetCheck(objectKey, s3MD5, gcpMD5, location, callback) { s3.send(new GetObjectCommand(params)) .then(res => { assert.strictEqual(res.ETag, `"${s3MD5}"`); - const metadataLocation = res.Metadata && - res.Metadata['scal-location-constraint']; + const metadataLocation = res.Metadata && res.Metadata['scal-location-constraint']; assert.strictEqual(metadataLocation, location); process.stdout.write('Getting object from GCP\n'); checkGcp(objectKey, gcpMD5, location, callback); @@ -83,22 +79,19 @@ function gcpGetCheck(objectKey, s3MD5, gcpMD5, location, callback) { }, retryTimeout); return; } - assert.strictEqual(err, null, 'Expected success, got error ' + - `on call to GCP through S3: ${err}`); + assert.strictEqual(err, null, 'Expected success, got error ' + `on call to GCP through S3: ${err}`); }); } attempt(); } -describeSkipIfNotMultiple('MultipleBackend put object to GCP', function -describeFn() { +describeSkipIfNotMultiple('MultipleBackend put object to GCP', function describeFn() { this.timeout(250000); withV4(sigCfg => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucket })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -106,19 +99,19 @@ describeFn() { afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); - describe('with set location from "x-amz-meta-scal-' + - 'location-constraint" header', function describe() { + describe('with set location from "x-amz-meta-scal-' + 'location-constraint" header', function describe() { if (!process.env.S3_END_TO_END) { this.retries(2); } @@ -145,13 +138,16 @@ describeFn() { const { s3MD5, gcpMD5 } = test.output; it(test.msg, done => { const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, Body, + const params = { + Bucket: bucket, + Key: key, + Body, Metadata: { 'scal-location-constraint': location }, }; - return s3.send(new PutObjectCommand(params)) + return s3 + .send(new PutObjectCommand(params)) .then(() => { - gcpGetCheck(key, s3MD5, gcpMD5, location, - () => done()); + gcpGetCheck(key, s3MD5, gcpMD5, location, () => done()); }) .catch(done); }); @@ -163,71 +159,82 @@ describeFn() { this.retries(2); } - it('should put objects with same key to GCP ' + - 'then file, and object should only be present in file', done => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': gcpLocation } }; - return s3.send(new PutObjectCommand(params)) - .then(() => { - params.Metadata = - { 'scal-location-constraint': fileLocation }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => s3.send(new GetObjectCommand({ + it( + 'should put objects with same key to GCP ' + 'then file, and object should only be present in file', + done => { + const key = `somekey-${genUniqID()}`; + const params = { Bucket: bucket, Key: key, - }))) - .then(res => { - assert.strictEqual( - res.Metadata['scal-location-constraint'], - fileLocation); - checkGcpError(key, 'NoSuchKey', - () => done()); - }) - .catch(done); - }); + Body: body, + Metadata: { 'scal-location-constraint': gcpLocation }, + }; + return s3 + .send(new PutObjectCommand(params)) + .then(() => { + params.Metadata = { 'scal-location-constraint': fileLocation }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ), + ) + .then(res => { + assert.strictEqual(res.Metadata['scal-location-constraint'], fileLocation); + checkGcpError(key, 'NoSuchKey', () => done()); + }) + .catch(done); + }, + ); - it('should put objects with same key to file ' + - 'then GCP, and object should only be present on GCP', done => { - const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, - Body: body, - Metadata: { 'scal-location-constraint': fileLocation } }; - return s3.send(new PutObjectCommand(params)) - .then(() => { - params.Metadata = { - 'scal-location-constraint': gcpLocation }; - return s3.send(new PutObjectCommand(params)); - }) - .then(() => { - gcpGetCheck(key, correctMD5, correctMD5, - gcpLocation, () => done()); - }) - .catch(done); - }); + it( + 'should put objects with same key to file ' + 'then GCP, and object should only be present on GCP', + done => { + const key = `somekey-${genUniqID()}`; + const params = { + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': fileLocation }, + }; + return s3 + .send(new PutObjectCommand(params)) + .then(() => { + params.Metadata = { + 'scal-location-constraint': gcpLocation, + }; + return s3.send(new PutObjectCommand(params)); + }) + .then(() => { + gcpGetCheck(key, correctMD5, correctMD5, gcpLocation, () => done()); + }) + .catch(done); + }, + ); - it('should put two objects to GCP with same ' + - 'key, and newest object should be returned', done => { + it('should put two objects to GCP with same ' + 'key, and newest object should be returned', done => { const key = `somekey-${genUniqID()}`; - const params = { Bucket: bucket, Key: key, + const params = { + Bucket: bucket, + Key: key, Body: body, - Metadata: { 'scal-location-constraint': gcpLocation, - 'unique-header': 'first object' } }; - return s3.send(new PutObjectCommand(params)) + Metadata: { 'scal-location-constraint': gcpLocation, 'unique-header': 'first object' }, + }; + return s3 + .send(new PutObjectCommand(params)) .then(() => { - params.Metadata = { 'scal-location-constraint': gcpLocation, - 'unique-header': 'second object' }; + params.Metadata = { 'scal-location-constraint': gcpLocation, 'unique-header': 'second object' }; return s3.send(new PutObjectCommand(params)); }) .then(() => { - gcpGetCheck(key, correctMD5, correctMD5, - gcpLocation, result => { - assert.strictEqual(result.Metadata - ['unique-header'], 'second object'); - done(); - }); + gcpGetCheck(key, correctMD5, correctMD5, gcpLocation, result => { + assert.strictEqual(result.Metadata['unique-header'], 'second object'); + done(); + }); }) .catch(done); }); @@ -244,33 +251,37 @@ describeSkipIfNotMultiple('MultipleBackend put object based on bucket location', afterEach(() => { process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + }); }); it('should put an object to GCP with no location header', done => { process.stdout.write('Creating bucket\n'); const key = `somekey-${genUniqID()}`; const params = { Bucket: bucket, Key: key, Body: body }; - return s3.send(new CreateBucketCommand({ Bucket: bucket, - CreateBucketConfiguration: { - LocationConstraint: gcpLocation, - }, - })) + return s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + CreateBucketConfiguration: { + LocationConstraint: gcpLocation, + }, + }), + ) .then(() => { process.stdout.write('Putting object\n'); return s3.send(new PutObjectCommand(params)); }) .then(() => { - gcpGetCheck(key, correctMD5, correctMD5, undefined, - () => done()); + gcpGetCheck(key, correctMD5, correctMD5, undefined, () => done()); }) .catch(done); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js b/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js index 9b8cae4a8d..d0331d95b1 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/unknownEndpoint.js @@ -32,39 +32,43 @@ describe('Requests to ip endpoint not in config', () => { await bucketUtil.deleteOne(bucket); }); - it('should accept put bucket request ' + - 'to IP address endpoint that is not in config using ' + - 'path style', + it( + 'should accept put bucket request ' + 'to IP address endpoint that is not in config using ' + 'path style', async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - }); + }, + ); const itSkipIfE2E = process.env.S3_END_TO_END ? it.skip : it; // skipping in E2E since in E2E 127.0.0.3 resolving to // localhost which is in config. Once integration is using // different machines we can update this. - itSkipIfE2E('should show us-east-1 as bucket location since' + - 'IP address endpoint was not in config thereby ' + - 'defaulting to us-east-1', + itSkipIfE2E( + 'should show us-east-1 as bucket location since' + + 'IP address endpoint was not in config thereby ' + + 'defaulting to us-east-1', async () => { const res = await s3.send(new GetBucketLocationCommand({ Bucket: bucket })); assert.strictEqual(res.LocationConstraint, undefined); - }); + }, + ); - it('should accept put object request ' + - 'to IP address endpoint that is not in config using ' + - 'path style and use the bucket location for the object', + it( + 'should accept put object request ' + + 'to IP address endpoint that is not in config using ' + + 'path style and use the bucket location for the object', async () => { await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: body })); await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); - }); + }, + ); - it('should accept get object request ' + - 'to IP address endpoint that is not in config using ' + - 'path style', + it( + 'should accept get object request ' + 'to IP address endpoint that is not in config using ' + 'path style', async () => { const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ETag, expectedETag); - }); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js index e87fc5058b..8f8b31555f 100644 --- a/tests/functional/aws-node-sdk/test/multipleBackend/utils.js +++ b/tests/functional/aws-node-sdk/test/multipleBackend/utils.js @@ -22,9 +22,7 @@ const { getRealAwsConfig } = require('../support/awsConfig'); const { config } = require('../../../../../lib/Config'); const authdata = require('../../../../../conf/authdata.json'); -const { - describeSkipIfNotMultiple, -} = require('../../lib/utility/test-utils'); +const { describeSkipIfNotMultiple } = require('../../lib/utility/test-utils'); const memLocation = 'scality-internal-mem'; const fileLocation = 'scality-internal-file'; @@ -56,23 +54,23 @@ if (config.backends.data === 'multiple') { awsS3 = new S3Client(awsConfig); awsBucket = config.locationConstraints[awsLocation].details.bucketName; } else { - process.stdout.write(`LocationConstraint for aws '${awsLocation}' not found in ${ - Object.keys(config.locationConstraints)}\n`); + process.stdout.write( + `LocationConstraint for aws '${awsLocation}' not found in ${Object.keys(config.locationConstraints)}\n`, + ); } if (config.locationConstraints[gcpLocation]) { const gcpConfig = getRealAwsConfig(gcpLocation); gcpClient = new GCP(gcpConfig); gcpBucket = config.locationConstraints[gcpLocation].details.bucketName; - gcpBucketMPU = - config.locationConstraints[gcpLocation].details.mpuBucketName; + gcpBucketMPU = config.locationConstraints[gcpLocation].details.mpuBucketName; } else { - process.stdout.write(`LocationConstraint for gcp '${gcpLocation}' not found in ${ - Object.keys(config.locationConstraints)}\n`); + process.stdout.write( + `LocationConstraint for gcp '${gcpLocation}' not found in ${Object.keys(config.locationConstraints)}\n`, + ); } } - const utils = { describeSkipIfNotMultiple, awsS3, @@ -137,11 +135,12 @@ utils.getAzureClient = () => { return true; } - if (config.locationConstraints[azureLocation] && + if ( + config.locationConstraints[azureLocation] && config.locationConstraints[azureLocation].details && - config.locationConstraints[azureLocation].details[key]) { - params[key] = - config.locationConstraints[azureLocation].details[key]; + config.locationConstraints[azureLocation].details[key] + ) { + params[key] = config.locationConstraints[azureLocation].details[key]; return true; } return false; @@ -151,20 +150,18 @@ utils.getAzureClient = () => { return undefined; } - const cred = new azure.StorageSharedKeyCredential( - params.azureStorageAccountName, - params.azureStorageAccessKey, - ); + const cred = new azure.StorageSharedKeyCredential(params.azureStorageAccountName, params.azureStorageAccessKey); return new azure.BlobServiceClient(params.azureStorageEndpoint, cred); }; utils.getAzureContainerName = azureLocation => { let azureContainerName; - if (config.locationConstraints[azureLocation] && - config.locationConstraints[azureLocation].details && - config.locationConstraints[azureLocation].details.azureContainerName) { - azureContainerName = - config.locationConstraints[azureLocation].details.azureContainerName; + if ( + config.locationConstraints[azureLocation] && + config.locationConstraints[azureLocation].details && + config.locationConstraints[azureLocation].details.azureContainerName + ) { + azureContainerName = config.locationConstraints[azureLocation].details.azureContainerName; } return azureContainerName; }; @@ -195,8 +192,7 @@ utils.getAzureKeys = () => { // For contentMD5, Azure requires base64 but AWS requires hex, so convert // from base64 to hex -utils.convertMD5 = contentMD5 => - Buffer.from(contentMD5, 'base64').toString('hex'); +utils.convertMD5 = contentMD5 => Buffer.from(contentMD5, 'base64').toString('hex'); utils.expectedETag = (body, getStringified = true) => { const eTagValue = crypto.createHash('md5').update(body).digest('hex'); @@ -215,9 +211,11 @@ utils.waitForVersioningBeforePut = async (s3, bucket, callback) => { for (let attempt = 1; attempt <= MAX_VERSIONING_CHECKS; attempt++) { let versioningEnabled = false; try { - const versioningResult = await s3.send(new GetBucketVersioningCommand({ - Bucket: bucket, - })); + const versioningResult = await s3.send( + new GetBucketVersioningCommand({ + Bucket: bucket, + }), + ); versioningEnabled = versioningResult.Status === 'Enabled'; } catch { if (attempt === MAX_VERSIONING_CHECKS) { @@ -244,34 +242,44 @@ utils.waitForVersioningBeforePut = async (s3, bucket, callback) => { }; utils.putToAwsBackend = (s3, bucket, key, body, callback) => { - const result = s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - Metadata: { 'scal-location-constraint': awsLocation } - })); + const result = s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + Metadata: { 'scal-location-constraint': awsLocation }, + }), + ); if (callback) { - return result.then(data => { - callback(null, data.VersionId); - }).catch(err => { - s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - Prefix: key, - })).then(data => { - callback(err, data.VersionId); - }).catch(listErr => { - callback(listErr); + return result + .then(data => { + callback(null, data.VersionId); + }) + .catch(err => { + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + Prefix: key, + }), + ) + .then(data => { + callback(err, data.VersionId); + }) + .catch(listErr => { + callback(listErr); + }); }); - }); } return result; }; utils.enableVersioning = (s3, bucket, callback) => { - const promise = s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled - })); + const promise = s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); if (callback) { return promise.then(() => callback()).catch(err => callback(err)); @@ -280,11 +288,13 @@ utils.enableVersioning = (s3, bucket, callback) => { }; utils.suspendVersioning = (s3, bucket, callback) => { - const promise = s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended - })); - + const promise = s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); + if (callback) { return promise.then(() => callback()).catch(err => callback(err)); } @@ -355,8 +365,7 @@ utils.putNullVersionsToAws = async (s3, bucket, key, versions, callback) => { utils.getAndAssertResult = (s3, params, callback) => { const run = async () => { - const { bucket, key, body, versionId, expectedVersionId, - expectedTagCount, expectedError } = params; + const { bucket, key, body, versionId, expectedVersionId, expectedTagCount, expectedError } = params; const getParams = { Bucket: bucket, Key: key, @@ -379,20 +388,25 @@ utils.getAndAssertResult = (s3, params, callback) => { chunks.push(chunk); } const bodyBuffer = Buffer.concat(chunks); - assert.equal(bodyBuffer.length, data.ContentLength, + assert.equal( + bodyBuffer.length, + data.ContentLength, `received data of length ${bodyBuffer.length} does not ` + - 'equal expected based on ' + - `content length header of ${data.ContentLength}`); + 'equal expected based on ' + + `content length header of ${data.ContentLength}`, + ); const expectedMD5 = utils.expectedETag(body, false); const resultMD5 = utils.expectedETag(bodyBuffer, false); assert.strictEqual(resultMD5, expectedMD5); } if (!expectedVersionId) { - assert.strictEqual(data.VersionId, undefined, - `Expected undefined VersionId but got ${data.VersionId}`); + assert.strictEqual(data.VersionId, undefined, `Expected undefined VersionId but got ${data.VersionId}`); } else { - assert.strictEqual(data.VersionId, expectedVersionId, - `Expected VersionId ${expectedVersionId} but got ${data.VersionId}`); + assert.strictEqual( + data.VersionId, + expectedVersionId, + `Expected VersionId ${expectedVersionId} but got ${data.VersionId}`, + ); } if (expectedTagCount && expectedTagCount === '0') { assert.strictEqual(data.TagCount, undefined); @@ -425,11 +439,11 @@ utils.getAwsRetry = (params, retryNumber, assertCb) => { }; const maxRetries = 2; const timeout = retryTimeout[retryNumber]; - + const executeGet = async () => { try { - const params = { - Bucket: awsBucket, + const params = { + Bucket: awsBucket, Key: key, VersionId: versionId, }; @@ -439,7 +453,7 @@ utils.getAwsRetry = (params, retryNumber, assertCb) => { return { success: false, error: err }; } }; - + return setTimeout(() => { executeGet() .then(result => { @@ -461,9 +475,8 @@ utils.getAwsRetry = (params, retryNumber, assertCb) => { utils.awsGetLatestVerId = (key, body, cb) => utils.getAwsRetry({ key }, 0, async (err, result) => { - assert.strictEqual(err, null, 'Expected success ' + - `getting object from AWS, got error ${err}`); - + assert.strictEqual(err, null, 'Expected success ' + `getting object from AWS, got error ${err}`); + const chunks = []; for await (const chunk of result.Body) { chunks.push(chunk); @@ -492,19 +505,21 @@ function _getTaggingConfig(tags) { utils.tagging.putTaggingAndAssert = async (s3, params) => { const { bucket, key, tags, versionId, expectedVersionId, expectedError } = params; const taggingConfig = _getTaggingConfig(tags); - + try { - const data = await s3.send(new PutObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - Tagging: taggingConfig - })); - + const data = await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + Tagging: taggingConfig, + }), + ); + if (expectedError) { throw new Error(`Expected error ${expectedError} but got success`); } - + if (expectedVersionId) { assert.strictEqual(data.VersionId, expectedVersionId); } else { @@ -521,35 +536,35 @@ utils.tagging.putTaggingAndAssert = async (s3, params) => { }; utils.tagging.getTaggingAndAssert = async (s3, params) => { - const { bucket, key, expectedTags, versionId, expectedVersionId, - expectedError, getObject } = params; - + const { bucket, key, expectedTags, versionId, expectedVersionId, expectedError, getObject } = params; + try { - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })); - + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ); + if (expectedError) { throw new Error(`Expected error ${expectedError} but got success`); } - + const expectedTagResult = _getTaggingConfig(expectedTags); const expectedTagCount = `${Object.keys(expectedTags).length}`; - + if (expectedVersionId) { assert.strictEqual(data.VersionId, expectedVersionId); } else { assert.strictEqual(data.VersionId, undefined); } assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet); - + if (getObject !== false) { - await utils.getAndAssertResult(s3, { bucket, key, versionId, - expectedVersionId, expectedTagCount }); + await utils.getAndAssertResult(s3, { bucket, key, versionId, expectedVersionId, expectedTagCount }); } - + return data.VersionId; } catch (err) { if (expectedError) { @@ -562,26 +577,32 @@ utils.tagging.getTaggingAndAssert = async (s3, params) => { utils.tagging.delTaggingAndAssert = async (s3, params) => { const { bucket, key, versionId, expectedVersionId, expectedError } = params; - + try { - const data = await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId - })); - + const data = await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ); + if (expectedError) { throw new Error(`Expected error ${expectedError} but got success`); } - + if (expectedVersionId) { assert.strictEqual(data.VersionId, expectedVersionId); } else { assert.strictEqual(data.VersionId, undefined); } - - await utils.tagging.getTaggingAndAssert(s3, { - bucket, key, versionId, expectedVersionId, expectedTags: {} + + await utils.tagging.getTaggingAndAssert(s3, { + bucket, + key, + versionId, + expectedVersionId, + expectedTags: {}, }); return undefined; } catch (err) { @@ -596,13 +617,15 @@ utils.tagging.delTaggingAndAssert = async (s3, params) => { utils.tagging.awsGetAssertTags = async params => { const { key, versionId, expectedTags } = params; const expectedTagResult = _getTaggingConfig(expectedTags); - - const data = await awsS3.send(new GetObjectTaggingCommand({ - Bucket: awsBucket, - Key: key, - VersionId: versionId - })); - + + const data = await awsS3.send( + new GetObjectTaggingCommand({ + Bucket: awsBucket, + Key: key, + VersionId: versionId, + }), + ); + assert.deepStrictEqual(data.TagSet, expectedTagResult.TagSet); }; diff --git a/tests/functional/aws-node-sdk/test/object/100-continue.js b/tests/functional/aws-node-sdk/test/object/100-continue.js index 9a41210f92..e43b40ba50 100644 --- a/tests/functional/aws-node-sdk/test/object/100-continue.js +++ b/tests/functional/aws-node-sdk/test/object/100-continue.js @@ -45,7 +45,7 @@ class ContinueRequestHandler { method: 'PUT', headers: { 'content-length': body.length, - 'Expect': this.expectHeader, + Expect: this.expectHeader, }, }; } @@ -102,8 +102,7 @@ class ContinueRequestHandler { req.flushHeaders(); // At this point we have only sent the header. const headerLen = req._header.length; - req.on('continue', () => - cb('Continue beeing seen when 403 is expected')); + req.on('continue', () => cb('Continue beeing seen when 403 is expected')); req.on('response', res => { res.on('data', () => {}); res.on('end', () => { @@ -138,30 +137,22 @@ describeSkipIfE2E('PUT public object with 100-continue header', () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); }); - afterEach(() => - bucketUtil.empty(bucket) - .then(() => bucketUtil.deleteOne(bucket))); + afterEach(() => bucketUtil.empty(bucket).then(() => bucketUtil.deleteOne(bucket))); - it('should return 200 status code', done => - continueRequest.hasStatusCode(200, done)); + it('should return 200 status code', done => continueRequest.hasStatusCode(200, done)); it('should return 200 status code with upper case value', done => - continueRequest.setExpectHeader('100-CONTINUE') - .hasStatusCode(200, done)); + continueRequest.setExpectHeader('100-CONTINUE').hasStatusCode(200, done)); it('should return 200 status code if incorrect value', done => - continueRequest.setExpectHeader('101-continue') - .hasStatusCode(200, done)); + continueRequest.setExpectHeader('101-continue').hasStatusCode(200, done)); it('should return 403 status code if cannot authenticate', done => - continueRequest.setRequestPath(invalidSignedURL) - .hasStatusCode(403, done)); + continueRequest.setRequestPath(invalidSignedURL).hasStatusCode(403, done)); - it('should wait for continue event before sending body', done => - continueRequest.sendsBodyOnContinue(done)); + it('should wait for continue event before sending body', done => continueRequest.sendsBodyOnContinue(done)); it('should not send continue if denied for a public user', done => - continueRequest.setRequestPath(invalidSignedURL) - .shouldNotGetContinue(done)); + continueRequest.setRequestPath(invalidSignedURL).shouldNotGetContinue(done)); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/abortMPU.js b/tests/functional/aws-node-sdk/test/object/abortMPU.js index 7c3a836548..6db6283340 100644 --- a/tests/functional/aws-node-sdk/test/object/abortMPU.js +++ b/tests/functional/aws-node-sdk/test/object/abortMPU.js @@ -8,7 +8,7 @@ const async = require('async'); const { initMetadata, getMetadata } = require('../utils/init'); const metadata = require('../../../../../lib/metadata/wrapper'); const { DummyRequestLogger } = require('../../../../unit/helpers'); -const { +const { CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, @@ -20,7 +20,7 @@ const { DeleteObjectCommand, PutBucketVersioningCommand, HeadObjectCommand, - PutObjectCommand + PutObjectCommand, } = require('@aws-sdk/client-s3'); const date = Date.now(); @@ -38,17 +38,23 @@ async function cleanupVersionedBucket(bucketUtil, bucketName) { // Clean up all multipart uploads const listMPUResponse = await bucketUtil.s3.send(new ListMultipartUploadsCommand({ Bucket: bucketName })); if (listMPUResponse.Uploads && listMPUResponse.Uploads.length > 0) { - await Promise.all(listMPUResponse.Uploads.map(async upload => { - bucketUtil.s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: upload.Key, - UploadId: upload.UploadId, - })).catch(err => { - if (err.name !== 'NoSuchUpload') { - throw err; - } - }); - })); + await Promise.all( + listMPUResponse.Uploads.map(async upload => { + bucketUtil.s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: upload.Key, + UploadId: upload.UploadId, + }), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload') { + throw err; + } + }); + }), + ); } // Clean up all object versions @@ -67,15 +73,22 @@ describe('Abort MPU', () => { s3 = bucketUtil.s3; try { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const mpu = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - })); + const mpu = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = mpu.UploadId; - await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, - PartNumber: 1, UploadId: uploadId, Body: bodyFirstPart, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: bodyFirstPart, + }), + ); } catch (err) { process.stdout.write(`Error in beforeEach: ${err}\n`); throw err; @@ -83,32 +96,35 @@ describe('Abort MPU', () => { }); afterEach(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); await bucketUtil.empty(bucket); await bucketUtil.deleteOne(bucket); }); // aws-sdk now (v2.363.0) returns 'UriParameterError' error // this test was not replaced in any other suite - it.skip('should return InvalidRequest error if aborting without key', - done => { - s3.send(new AbortMultipartUploadCommand({ + it.skip('should return InvalidRequest error if aborting without key', done => { + s3.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: '', - UploadId: uploadId - })) - .then(() => { - done(new Error('Expected failure but got success')); - }) - .catch(err => { - checkError(err, 'InvalidRequest', 'A key must be specified'); - done(); - }); - }); + UploadId: uploadId, + }), + ) + .then(() => { + done(new Error('Expected failure but got success')); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 'A key must be specified'); + done(); + }); + }); }); }); @@ -137,86 +153,105 @@ describe('Abort MPU with existing object', function AbortMPUExistingObject() { let uploadId1; let uploadId2; let etag1; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId1 = data.UploadId; - return s3.send(new UploadPartCommand({ + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, - UploadId: uploadId1, - Body: part1, - })); - }) - .then(data => { - etag1 = data.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + }), + ) + .then(data => { + uploadId1 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId1, + Body: part1, + }), + ); + }) + .then(data => { + etag1 = data.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId1, + MultipartUpload: { Parts: [{ ETag: etag1, PartNumber: 1 }] }, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ Bucket: bucketName, Key: objectKey, - UploadId: uploadId1, - MultipartUpload: { Parts: [{ ETag: etag1, PartNumber: 1 }] }, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part1.toString()); - next(); - }) - .catch(err => next(err)); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId2 = data.UploadId; - return s3.send(new UploadPartCommand({ + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part1.toString()); + next(); + }) + .catch(err => next(err)); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(data => { + uploadId2 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId2, + Body: part2, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new AbortMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, UploadId: uploadId2, - Body: part2, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId2, - })) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part1.toString()); - next(); - }) - .catch(err => next(err)); - }, - ], done); + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part1.toString()); + next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); it('should not delete existing object data when aborting an old MPU for same key', done => { @@ -225,86 +260,105 @@ describe('Abort MPU with existing object', function AbortMPUExistingObject() { let uploadId1; let uploadId2; let etag2; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId1 = data.UploadId; - return s3.send(new UploadPartCommand({ + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, - UploadId: uploadId1, - Body: part1, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(data => { - uploadId2 = data.UploadId; - return s3.send(new UploadPartCommand({ + }), + ) + .then(data => { + uploadId1 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId1, + Body: part1, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - PartNumber: 1, - UploadId: uploadId2, - Body: part2, - })); - }) - .then(data => { - etag2 = data.ETag; - return s3.send(new CompleteMultipartUploadCommand({ + }), + ) + .then(data => { + uploadId2 = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId2, + Body: part2, + }), + ); + }) + .then(data => { + etag2 = data.ETag; + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId2, + MultipartUpload: { Parts: [{ ETag: etag2, PartNumber: 1 }] }, + }), + ); + }) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ Bucket: bucketName, Key: objectKey, - UploadId: uploadId2, - MultipartUpload: { Parts: [{ ETag: etag2, PartNumber: 1 }] }, - })); - }) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part2.toString()); - next(); - }) - .catch(err => next(err)); - }, - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId1, - })) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(async data => { - const bodyText = await data.Body.transformToString(); - assert.strictEqual(bodyText, part2.toString()); - next(); - }) - .catch(err => next(err)); - }, - ], done); + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part2.toString()); + next(); + }) + .catch(err => next(err)); + }, + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId1, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(async data => { + const bodyText = await data.Body.transformToString(); + assert.strictEqual(bodyText, part2.toString()); + next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); }); }); @@ -322,22 +376,23 @@ describe('Abort MPU - No Such Upload', () => { afterEach(() => bucketUtil.deleteOne(bucket)); - it('should return NoSuchUpload error when aborting non-existent mpu', - done => { - s3.send(new AbortMultipartUploadCommand({ + it('should return NoSuchUpload error when aborting non-existent mpu', done => { + s3.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, - UploadId: uuidv4().replace(/-/g, '') - })) - .then(() => { - done(new Error('Expected failure but got success')); - }) - .catch(err => { - assert.notEqual(err, null, 'Expected failure but got success'); - assert.strictEqual(err.name, 'NoSuchUpload'); - done(); - }); - }); + UploadId: uuidv4().replace(/-/g, ''), + }), + ) + .then(() => { + done(new Error('Expected failure but got success')); + }) + .catch(err => { + assert.notEqual(err, null, 'Expected failure but got success'); + assert.strictEqual(err.name, 'NoSuchUpload'); + done(); + }); + }); }); }); @@ -355,10 +410,12 @@ describe('Abort MPU - Versioned Bucket Cleanup', function testSuite() { s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); }); afterEach(async () => { @@ -377,68 +434,83 @@ describe('Abort MPU - Versioned Bucket Cleanup', function testSuite() { currentVersion++; const data = Buffer.from(`Version ${currentVersion} data`); - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(result => { - if (currentVersion === numberOfVersions) { - finalUploadId = result.UploadId; // Save the last one for aborting - } - next(null, result.UploadId); - }) - .catch(err => next(err)); - }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })) - .then(result => next(null, uploadId, result.ETag)) - .catch(err => next(err)); - }, - (uploadId, etag, next) => { - if (currentVersion === numberOfVersions) { - // Don't complete the last one - we'll abort it - return next(); - } - - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ ETag: etag, PartNumber: 1 }], - }, - })) - .then(() => next()) - .catch(err => next(err)); - }, - ], callback); + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(result => { + if (currentVersion === numberOfVersions) { + finalUploadId = result.UploadId; // Save the last one for aborting + } + next(null, result.UploadId); + }) + .catch(err => next(err)); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ) + .then(result => next(null, uploadId, result.ETag)) + .catch(err => next(err)); + }, + (uploadId, etag, next) => { + if (currentVersion === numberOfVersions) { + // Don't complete the last one - we'll abort it + return next(); + } + + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + ], + callback, + ); }, err => { assert.ifError(err); // Now abort the final MPU - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: finalUploadId, - })) + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: finalUploadId, + }), + ) .then(() => s3.send(new ListObjectVersionsCommand({ Bucket: bucketName }))) .then(data => { const objectVersions = data.Versions.filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, numberOfVersions - 1, - `Expected ${numberOfVersions - 1} versions after abort, got ${objectVersions.length}`); + assert.strictEqual( + objectVersions.length, + numberOfVersions - 1, + `Expected ${numberOfVersions - 1} versions after abort, got ${objectVersions.length}`, + ); done(); }) .catch(err => done(err)); - } + }, ); }); @@ -446,69 +518,83 @@ describe('Abort MPU - Versioned Bucket Cleanup', function testSuite() { let uploadId; const data = Buffer.from('test data for single MPU abort'); - async.waterfall([ - // Create and upload part for MPU - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(result => { - uploadId = result.UploadId; - next(); - }) - .catch(err => next(err)); - }, - next => { - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })) - .then(() => next()) - .catch(err => next(err)); - }, + async.waterfall( + [ + // Create and upload part for MPU + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(result => { + uploadId = result.UploadId; + next(); + }) + .catch(err => next(err)); + }, + next => { + s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, - // Abort the MPU - next => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })) - .then(() => next()) - .catch(err => next(err)); - }, + // Abort the MPU + next => { + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, - // Verify no object exists - next => { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(() => { - next(new Error('Expected NoSuchKey error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NoSuchKey'); - next(); - }); - }, + // Verify no object exists + next => { + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(() => { + next(new Error('Expected NoSuchKey error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchKey'); + next(); + }); + }, - // Verify no versions exist - next => { - s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })) - .then(data => { - const objectVersions = (data.Versions || []).filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, 0, - `Expected 0 versions after abort, got ${objectVersions.length}`); - next(); - }) - .catch(err => next(err)); - }, - ], done); + // Verify no versions exist + next => { + s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })) + .then(data => { + const objectVersions = (data.Versions || []).filter(v => v.Key === objectKey); + assert.strictEqual( + objectVersions.length, + 0, + `Expected 0 versions after abort, got ${objectVersions.length}`, + ); + next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); }); }); @@ -531,32 +617,44 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { * @param {boolean} isVersioned - Whether to create versioned metadata * @returns {Promise} Promise that resolves when orphaned metadata is created */ - async function createOrphanedObjectMetadata(s3Client, bucketName, objectKey, uploadIdToSimulate, - data, isVersioned) { + async function createOrphanedObjectMetadata( + s3Client, + bucketName, + objectKey, + uploadIdToSimulate, + data, + isVersioned, + ) { const tempObjectKey = `temp-object-for-metadata-${Date.now()}`; // Create temporary MPU and complete it to get real object metadata - const createResult = await s3Client.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: tempObjectKey, - })); + const createResult = await s3Client.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: tempObjectKey, + }), + ); const tempUploadId = createResult.UploadId; - const uploadResult = await s3Client.send(new UploadPartCommand({ - Bucket: bucketName, - Key: tempObjectKey, - PartNumber: 1, - UploadId: tempUploadId, - Body: data, - })); + const uploadResult = await s3Client.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: tempObjectKey, + PartNumber: 1, + UploadId: tempUploadId, + Body: data, + }), + ); const tempEtag = uploadResult.ETag; - const completeResult = await s3Client.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: tempObjectKey, - UploadId: tempUploadId, - MultipartUpload: { Parts: [{ ETag: tempEtag, PartNumber: 1 }] }, - })); + const completeResult = await s3Client.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: tempObjectKey, + UploadId: tempUploadId, + MultipartUpload: { Parts: [{ ETag: tempEtag, PartNumber: 1 }] }, + }), + ); let tempVersionId; if (isVersioned && completeResult.VersionId) { @@ -569,14 +667,15 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { // Create a copy and override uploadId to match our test MPU // (simulating orphaned object) - const orphanedObjectMD = Object.assign({}, objMD, + const orphanedObjectMD = Object.assign( + {}, + objMD, // let metadata generate a new versionId - { uploadId: uploadIdToSimulate, versionId: undefined }); + { uploadId: uploadIdToSimulate, versionId: undefined }, + ); // Store this modified metadata as orphaned object - const putOptions = isVersioned && objMD.versionId - ? { versioning: true } - : {}; + const putOptions = isVersioned && objMD.versionId ? { versioning: true } : {}; await putObjectMDAsync(bucketName, objectKey, orphanedObjectMD, putOptions, log); // Clean up temporary object @@ -600,11 +699,17 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - async.series([ - next => s3.send(new CreateBucketCommand({ Bucket: bucketName })).then(() => - next()).catch(err => next(err)), - next => initMetadata(next), - ], done); + async.series( + [ + next => + s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) + .then(() => next()) + .catch(err => next(err)), + next => initMetadata(next), + ], + done, + ); }); afterEach(async () => { @@ -615,19 +720,23 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { const data = Buffer.from('test data for orphan cleanup'); // Create MPU and upload a part - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); const uploadId = createResult.UploadId; - await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ); // Create realistic orphaned object metadata like a CompleteMPU would when failing before cleanup await createOrphanedObjectMetadata(s3, bucketName, objectKey, uploadId, data, false); @@ -636,11 +745,13 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { await s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectKey })); // Abort MPU - should clean up the orphaned object - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ); // Verify the orphaned object was cleaned up try { @@ -657,63 +768,85 @@ describe('Abort MPU - Orphan Cleanup', function testSuite() { const data = Buffer.from('test versioned orphan cleanup'); // Enable versioning - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); // Create MPU - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); const uploadId = createResult.UploadId; - await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ); // Create realistic orphaned object metadata like a CompleteMPU would when failing before cleanup const orphanedObjectMD = await createOrphanedObjectMetadata( - s3, bucketName, objectKey, uploadId, data, true); + s3, + bucketName, + objectKey, + uploadId, + data, + true, + ); // Put a new master version on top of the orphaned version // The abort will fetch this during standardMetadataValidateBucketAndObj // It will force abort to findObjectVersionByUploadId - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - Body: 'version 2 data', - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectKey, + Body: 'version 2 data', + }), + ); // Verify we have 2 versions (1 regular + 1 orphaned) let listResult = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); let objectVersions = listResult.Versions.filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, 2, - 'Expected 2 versions before abort, 1 regular + 1 orphaned' - ); + assert.strictEqual(objectVersions.length, 2, 'Expected 2 versions before abort, 1 regular + 1 orphaned'); // Abort MPU - should find and clean up only the orphaned version - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ); // Verify only the orphaned version was deleted listResult = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); objectVersions = listResult.Versions.filter(v => v.Key === objectKey); - assert.strictEqual(objectVersions.length, 1, - 'Should have 1 version after abort (orphaned version cleaned up)'); + assert.strictEqual( + objectVersions.length, + 1, + 'Should have 1 version after abort (orphaned version cleaned up)', + ); // ensure orphanedObj doesn't exist try { - await s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectKey, - VersionId: orphanedObjectMD.versionId })); + await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectKey, + VersionId: orphanedObjectMD.versionId, + }), + ); assert.fail('Orphaned object should be deleted after abort'); } catch (err) { assert(err); @@ -746,38 +879,48 @@ describe('Abort MPU - Race Conditions', function testSuite() { const data = Buffer.from('test concurrent complete and abort'); // Create MPU and upload part - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); + const createResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ); const uploadId = createResult.UploadId; - const uploadResult = await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); + const uploadResult = await s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: objectKey, + PartNumber: 1, + UploadId: uploadId, + Body: data, + }), + ); const etag = uploadResult.ETag; // Start concurrent operations: CompleteMPU and AbortMPU const [completeResult, abortResult] = await Promise.allSettled([ - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ ETag: etag, PartNumber: 1 }], - }, - })), + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ), // Add small delay to create race condition - scheduler.wait(10).then(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - }))) + scheduler.wait(10).then(() => + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), + ), ]); // Verify final state is consistent @@ -821,43 +964,51 @@ describe('Abort MPU - Race Conditions', function testSuite() { const data = Buffer.from('test multiple concurrent aborts'); // Create MPU and upload part - const createResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - })); - const uploadId = createResult.UploadId; - - await s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: objectKey, - PartNumber: 1, - UploadId: uploadId, - Body: data, - })); - - // Launch 3 concurrent abort operations - const abortResults = await Promise.allSettled([ - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: objectKey, - UploadId: uploadId, - })), - s3.send(new AbortMultipartUploadCommand({ + const createResult = await s3.send( + new CreateMultipartUploadCommand({ Bucket: bucketName, Key: objectKey, - UploadId: uploadId, - })), - s3.send(new AbortMultipartUploadCommand({ + }), + ); + const uploadId = createResult.UploadId; + + await s3.send( + new UploadPartCommand({ Bucket: bucketName, Key: objectKey, + PartNumber: 1, UploadId: uploadId, - })) + Body: data, + }), + ); + + // Launch 3 concurrent abort operations + const abortResults = await Promise.allSettled([ + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectKey, + UploadId: uploadId, + }), + ), ]); // Verify results - const abortErrors = abortResults.map(result => - result.status === 'rejected' ? result.reason : null - ); + const abortErrors = abortResults.map(result => (result.status === 'rejected' ? result.reason : null)); // At least one abort should succeed const successfulAborts = abortErrors.filter(err => !err); @@ -883,8 +1034,7 @@ describe('Abort MPU - Race Conditions', function testSuite() { // Verify no MPU metadata remains const listResult = await s3.send(new ListMultipartUploadsCommand({ Bucket: bucketName })); const remainingUploads = (listResult.Uploads || []).filter(upload => upload.UploadId === uploadId); - assert.strictEqual(remainingUploads.length, 0, - 'No MPU metadata should remain after concurrent aborts'); + assert.strictEqual(remainingUploads.length, 0, 'No MPU metadata should remain after concurrent aborts'); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/bigMpu.js b/tests/functional/aws-node-sdk/test/object/bigMpu.js index 0570531ea6..f4666fa936 100644 --- a/tests/functional/aws-node-sdk/test/object/bigMpu.js +++ b/tests/functional/aws-node-sdk/test/object/bigMpu.js @@ -2,15 +2,15 @@ const assert = require('assert'); const { timesLimit, waterfall } = require('async'); const { NodeHttpHandler } = require('@smithy/node-http-handler'); -const { +const { S3Client, CreateBucketCommand, - CreateMultipartUploadCommand, - UploadPartCommand, - CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + UploadPartCommand, + CompleteMultipartUploadCommand, GetObjectCommand, DeleteObjectCommand, - DeleteBucketCommand + DeleteBucketCommand, } = require('@aws-sdk/client-s3'); const getConfig = require('../support/config'); @@ -20,9 +20,10 @@ const key = 'mpuKey'; const body = 'abc'; const partCount = 10000; const eTag = require('crypto').createHash('md5').update(body).digest('hex'); -const finalETag = require('crypto').createHash('md5') - .update(Buffer.from(eTag.repeat(partCount), 'hex').toString('binary'), - 'binary').digest('hex'); +const finalETag = require('crypto') + .createHash('md5') + .update(Buffer.from(eTag.repeat(partCount), 'hex').toString('binary'), 'binary') + .digest('hex'); const partETags = new Array(partCount); function uploadPart(n, uploadId, s3, next) { @@ -36,7 +37,7 @@ function uploadPart(n, uploadId, s3, next) { if (params.PartNumber % 20 === 0) { process.stdout.write(`uploading PartNumber: ${params.PartNumber}\n`); } - + s3.send(new UploadPartCommand(params)) .then(data => { partETags[n] = data.ETag; @@ -65,13 +66,13 @@ describe('large mpu', function tester() { requestTimeout: 0, connectionTimeout: 0, }); - + s3 = new S3Client({ ...config, maxAttempts: 1, requestHandler, }); - + s3.send(new CreateBucketCommand({ Bucket: bucket })) .then(() => done()) .catch(err => done(err)); @@ -90,58 +91,66 @@ describe('large mpu', function tester() { const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; // will fail on AWS because parts too small - itSkipIfAWS('should intiate, put parts and complete mpu ' + - `with ${partCount} parts`, done => { + itSkipIfAWS('should intiate, put parts and complete mpu ' + `with ${partCount} parts`, done => { process.stdout.write('***Running large MPU test***\n'); - let uploadId; - return waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(err => next(err)); - }, - next => timesLimit(partCount, 20, (n, cb) => uploadPart(n, uploadId, s3, cb), err => { - if (err) { - process.stdout.write(`Error in timesLimit: ${err}\n`); - } - return next(err); - }), - next => { - const parts = []; - for (let i = 0; i < partCount; i++) { - if (!partETags[i]) { - return next(new Error(`Missing ETag for part ${i + 1}`)); + let uploadId; + return waterfall( + [ + next => { + s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: key })) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(err => next(err)); + }, + next => + timesLimit( + partCount, + 20, + (n, cb) => uploadPart(n, uploadId, s3, cb), + err => { + if (err) { + process.stdout.write(`Error in timesLimit: ${err}\n`); + } + return next(err); + }, + ), + next => { + const parts = []; + for (let i = 0; i < partCount; i++) { + if (!partETags[i]) { + return next(new Error(`Missing ETag for part ${i + 1}`)); + } + parts.push({ + ETag: partETags[i], + PartNumber: i + 1, + }); } - parts.push({ - ETag: partETags[i], - PartNumber: i + 1, - }); - } - const params = { - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { - Parts: parts, - }, - }; - return s3.send(new CompleteMultipartUploadCommand(params)) - .then(() => next()) - .catch(err => next(err)); - }, - next => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) - .then(data => { - assert.strictEqual(data.ETag, - `"${finalETag}-${partCount}"`); - process.stdout.write('get object successful\n'); - return next(); - }) - .catch(err => next(err)); - }, - ], done); + const params = { + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: parts, + }, + }; + return s3 + .send(new CompleteMultipartUploadCommand(params)) + .then(() => next()) + .catch(err => next(err)); + }, + next => { + s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + .then(data => { + assert.strictEqual(data.ETag, `"${finalETag}-${partCount}"`); + process.stdout.write('get object successful\n'); + return next(); + }) + .catch(err => next(err)); + }, + ], + done, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/completeMPU.js b/tests/functional/aws-node-sdk/test/object/completeMPU.js index f51a1d64c3..b78ca397b5 100644 --- a/tests/functional/aws-node-sdk/test/object/completeMPU.js +++ b/tests/functional/aws-node-sdk/test/object/completeMPU.js @@ -2,11 +2,7 @@ const assert = require('assert'); const async = require('async'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const { taggingTests } = require('../../lib/utility/tagging'); const { CreateBucketCommand, @@ -17,14 +13,13 @@ const { GetObjectCommand, PutBucketVersioningCommand, GetObjectTaggingCommand, - NoSuchKey + NoSuchKey, } = require('@aws-sdk/client-s3'); const date = Date.now(); const bucket = `completempu${date}`; const key = 'key'; - describe('Complete MPU', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -32,59 +27,68 @@ describe('Complete MPU', () => { function _completeMpuAndCheckVid(uploadId, eTag, expectedVid, cb) { let versionId; - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - MultipartUpload: { - Parts: [{ ETag: eTag, PartNumber: 1 }], - }, - UploadId: uploadId - })) - .then(data => { - versionId = data.VersionId; - if (expectedVid) { - assert.notEqual(versionId, undefined); - } else { - assert.strictEqual(versionId, expectedVid); - } - return s3.send(new GetObjectCommand({ + s3.send( + new CompleteMultipartUploadCommand({ Bucket: bucket, Key: key, - })); - }) - .then(data => { - if (versionId) { - assert.strictEqual(data.VersionId, versionId); - } - cb(); - }) - .catch(cb); + MultipartUpload: { + Parts: [{ ETag: eTag, PartNumber: 1 }], + }, + UploadId: uploadId, + }), + ) + .then(data => { + versionId = data.VersionId; + if (expectedVid) { + assert.notEqual(versionId, undefined); + } else { + assert.strictEqual(versionId, expectedVid); + } + return s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + }) + .then(data => { + if (versionId) { + assert.strictEqual(data.VersionId, versionId); + } + cb(); + }) + .catch(cb); } function _initiateMpuAndPutOnePart() { const result = {}; - return s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key - })) - .then(data => { - result.uploadId = data.UploadId; - return s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: data.UploadId, - Body: 'foo', - })); - }) - .then(data => { - result.eTag = data.ETag; - return result; - }) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - throw err; - }); + return s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(data => { + result.uploadId = data.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: data.UploadId, + Body: 'foo', + }), + ); + }) + .then(data => { + result.eTag = data.ETag; + return result; + }) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + throw err; + }); } beforeEach(async () => { @@ -109,59 +113,74 @@ describe('Complete MPU', () => { let uploadId; let eTag; - beforeEach(() => _initiateMpuAndPutOnePart() - .then(result => { + beforeEach(() => + _initiateMpuAndPutOnePart().then(result => { uploadId = result.uploadId; eTag = result.eTag; - }) + }), ); - it('should complete an MPU with fewer parts than were ' + - 'originally put without returning a version id', done => { - _completeMpuAndCheckVid(uploadId, eTag, undefined, done); - }); + it( + 'should complete an MPU with fewer parts than were ' + 'originally put without returning a version id', + done => { + _completeMpuAndCheckVid(uploadId, eTag, undefined, done); + }, + ); }); describe('on bucket with enabled versioning', () => { let uploadId; let eTag; - beforeEach(() => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled - })) - .then(() => _initiateMpuAndPutOnePart()) - .then(result => { - uploadId = result.uploadId; - eTag = result.eTag; - }) + beforeEach(() => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => _initiateMpuAndPutOnePart()) + .then(result => { + uploadId = result.uploadId; + eTag = result.eTag; + }), ); - it('should complete an MPU with fewer parts than were ' + - 'originally put and return a version id', done => { - _completeMpuAndCheckVid(uploadId, eTag, true, done); - }); + it( + 'should complete an MPU with fewer parts than were ' + 'originally put and return a version id', + done => { + _completeMpuAndCheckVid(uploadId, eTag, true, done); + }, + ); }); describe('on bucket with suspended versioning', () => { let uploadId; let eTag; - beforeEach(() => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended - })) - .then(() => _initiateMpuAndPutOnePart()) - .then(result => { - uploadId = result.uploadId; - eTag = result.eTag; - }) + beforeEach(() => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ) + .then(() => _initiateMpuAndPutOnePart()) + .then(result => { + uploadId = result.uploadId; + eTag = result.eTag; + }), ); - it('should complete an MPU with fewer parts than were ' + - 'originally put and should not return a version id', done => { - _completeMpuAndCheckVid(uploadId, eTag, undefined, done); - }); + it( + 'should complete an MPU with fewer parts than were ' + + 'originally put and should not return a version id', + done => { + _completeMpuAndCheckVid(uploadId, eTag, undefined, done); + }, + ); }); describe('with tags set on initiation', () => { @@ -169,78 +188,91 @@ describe('Complete MPU', () => { taggingTests.forEach(test => { it(test.it, done => { - const [key, value] = - [test.tag.key, test.tag.value].map(encodeURIComponent); + const [key, value] = [test.tag.key, test.tag.value].map(encodeURIComponent); const tagging = `${key}=${value}`; - async.waterfall([ - next => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: tagKey, - Tagging: tagging, - })) - .then(data => { - if (test.error) { - return next(new Error('Expected error but got success')); - } - return next(null, data.UploadId); - }) - .catch(err => { - if (test.error) { - assert.strictEqual(err.name, test.error); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - return next('expected'); - } - return next(err); - }); - }, - (uploadId, next) => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: tagKey, - PartNumber: 1, - UploadId: uploadId, - Body: 'foo', - })) - .then(data => next(null, data.ETag, uploadId)) - .catch(err => next(err)); - }, - (eTag, uploadId, next) => { - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: tagKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ - ETag: eTag, + async.waterfall( + [ + next => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: tagKey, + Tagging: tagging, + }), + ) + .then(data => { + if (test.error) { + return next(new Error('Expected error but got success')); + } + return next(null, data.UploadId); + }) + .catch(err => { + if (test.error) { + assert.strictEqual(err.name, test.error); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + return next('expected'); + } + return next(err); + }); + }, + (uploadId, next) => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: tagKey, PartNumber: 1, - }], - }, - })) - .then(() => next()) - .catch(err => next(err)); - }, - ], err => { - if (err === 'expected') { - done(); - } else { - assert.ifError(err); - s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: tagKey, - })) - .then(tagData => { - assert.deepStrictEqual(tagData.TagSet, - [{ - Key: test.tag.key, - Value: test.tag.value, - }]); + UploadId: uploadId, + Body: 'foo', + }), + ) + .then(data => next(null, data.ETag, uploadId)) + .catch(err => next(err)); + }, + (eTag, uploadId, next) => { + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: tagKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + ETag: eTag, + PartNumber: 1, + }, + ], + }, + }), + ) + .then(() => next()) + .catch(err => next(err)); + }, + ], + err => { + if (err === 'expected') { done(); - }) - .catch(err => done(err)); - } - }); + } else { + assert.ifError(err); + s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: tagKey, + }), + ) + .then(tagData => { + assert.deepStrictEqual(tagData.TagSet, [ + { + Key: test.tag.key, + Value: test.tag.value, + }, + ]); + done(); + }) + .catch(err => done(err)); + } + }, + ); }); }); }); @@ -249,37 +281,41 @@ describe('Complete MPU', () => { let uploadId; let eTag; - beforeEach(() => _initiateMpuAndPutOnePart() - .then(result => { + beforeEach(() => + _initiateMpuAndPutOnePart().then(result => { uploadId = result.uploadId; eTag = result.eTag; - }) + }), ); it('should complete the MPU successfully and leave a readable object', done => { - async.parallel([ - doneReUpload => { - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: 'foo', - })) - .then(() => doneReUpload()) - .catch(err => { - // in case the CompleteMPU finished earlier, - // we may get a NoSuchKey error, so just - // ignore it - if (err instanceof NoSuchKey) { - return doneReUpload(); - } - return doneReUpload(err); - }); - }, - doneComplete => _completeMpuAndCheckVid( - uploadId, eTag, undefined, doneComplete), - ], done); + async.parallel( + [ + doneReUpload => { + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: 'foo', + }), + ) + .then(() => doneReUpload()) + .catch(err => { + // in case the CompleteMPU finished earlier, + // we may get a NoSuchKey error, so just + // ignore it + if (err instanceof NoSuchKey) { + return doneReUpload(); + } + return doneReUpload(err); + }); + }, + doneComplete => _completeMpuAndCheckVid(uploadId, eTag, undefined, doneComplete), + ], + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/compluteMpu.js b/tests/functional/aws-node-sdk/test/object/compluteMpu.js index d8a20a6c7a..339b2407e0 100644 --- a/tests/functional/aws-node-sdk/test/object/compluteMpu.js +++ b/tests/functional/aws-node-sdk/test/object/compluteMpu.js @@ -47,13 +47,14 @@ describe('aws-node-sdk test bucket complete mpu', () => { Parts: parts, }, }; - s3.send(new CompleteMultipartUploadCommand(params)).then(() => { - done('accepted xml body larger than 1 MB'); - }).catch(error => { - assert.strictEqual(error.$metadata.httpStatusCode, 400); - assert.strictEqual( - error.name, 'InvalidRequest'); - done(); - }); + s3.send(new CompleteMultipartUploadCommand(params)) + .then(() => { + done('accepted xml body larger than 1 MB'); + }) + .catch(error => { + assert.strictEqual(error.$metadata.httpStatusCode, 400); + assert.strictEqual(error.name, 'InvalidRequest'); + done(); + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/copyPart.js b/tests/functional/aws-node-sdk/test/object/copyPart.js index 52aec224d9..302e4c9f63 100644 --- a/tests/functional/aws-node-sdk/test/object/copyPart.js +++ b/tests/functional/aws-node-sdk/test/object/copyPart.js @@ -1,7 +1,8 @@ const assert = require('assert'); const crypto = require('crypto'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, GetObjectCommand, HeadObjectCommand, @@ -10,13 +11,12 @@ const { CreateBucketCommand, UploadPartCopyCommand, CompleteMultipartUploadCommand, AbortMultipartUploadCommand, - PutObjectAclCommand + PutObjectAclCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { createEncryptedBucketPromise } = - require('../../lib/utility/createEncryptedBucket'); +const { createEncryptedBucketPromise } = require('../../lib/utility/createEncryptedBucket'); const { fakeMetadataTransition, fakeMetadataArchive } = require('../utils/init'); const { hasColdStorage } = require('../../lib/utility/test-utils'); @@ -33,8 +33,7 @@ const otherAccountS3 = otherAccountBucketUtility.s3; const oneHundredMBPlus11 = 110100481; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function checkError(err, code) { @@ -56,240 +55,350 @@ describe('Object Part Copy', () => { if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { s3.createBucketPromise = createEncryptedBucketPromise; } - return s3.createBucketPromise({ Bucket: sourceBucketName }) - .catch(err => { - process.stdout.write(`Error creating source bucket: ${err}\n`); - throw err; - }).then(() => - s3.createBucketPromise({ Bucket: destBucketName }) - ).catch(err => { - process.stdout.write(`Error creating dest bucket: ${err}\n`); - throw err; - }) - .then(() => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: content, - }))) - .then(res => { - etag = res.ETag; - return s3.send(new HeadObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - })); - }).then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - })).then(initiateRes => { - uploadId = initiateRes.UploadId; - })).catch(err => { - process.stdout.write(`Error in outer beforeEach: ${err}\n`); - throw err; - }); - }); - - afterEach(() => bucketUtil.empty(sourceBucketName) - .then(() => bucketUtil.empty(destBucketName)) - .then(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - }))).catch(err => { - if (err.name !== 'NoSuchUpload') { - process.stdout.write(`Error in afterEach: ${err}\n`); + return s3 + .createBucketPromise({ Bucket: sourceBucketName }) + .catch(err => { + process.stdout.write(`Error creating source bucket: ${err}\n`); throw err; - } - }) - .then(() => bucketUtil.deleteMany([sourceBucketName, - destBucketName])) - ); + }) + .then(() => s3.createBucketPromise({ Bucket: destBucketName })) + .catch(err => { + process.stdout.write(`Error creating dest bucket: ${err}\n`); + throw err; + }) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: content, + }), + ), + ) + .then(res => { + etag = res.ETag; + return s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + }), + ); + }) + .then(() => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(initiateRes => { + uploadId = initiateRes.UploadId; + }), + ) + .catch(err => { + process.stdout.write(`Error in outer beforeEach: ${err}\n`); + throw err; + }); + }); + afterEach(() => + bucketUtil + .empty(sourceBucketName) + .then(() => bucketUtil.empty(destBucketName)) + .then(() => + s3.send( + new AbortMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + }), + ), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload') { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + } + }) + .then(() => bucketUtil.deleteMany([sourceBucketName, destBucketName])), + ); - it('should copy a part from a source bucket to a different ' + - 'destination bucket', () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - })); - - it('should copy a part from a source bucket to a different ' + - 'destination bucket and complete the MPU', () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, + it('should copy a part from a source bucket to a different ' + 'destination bucket', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: etag, PartNumber: 1 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - // AWS confirmed final ETag for MPU - assert.strictEqual(res.ETag, - '"db77ebbae9e9f5a244a26b86193ad818-1"'); - }); - })); - - it('should return InvalidArgument error given invalid range', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: 'bad-range-parameter', - })).catch(err => { - checkError(err, 'InvalidArgument'); - }))); - - it('should return EntityTooLarge error if attempt to copy ' + - 'object larger than max and do not specify smaller ' + - 'range in request', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - }))).catch(err => { - checkError(err, 'EntityTooLarge'); - })); - - it('should return EntityTooLarge error if attempt to copy ' + - 'object larger than max and specify too large ' + - 'range in request', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: `bytes=0-${oneHundredMBPlus11}`, - }))).catch(err => { - checkError(err, 'EntityTooLarge'); - })); - - it('should succeed if attempt to copy ' + - 'object larger than max but specify acceptable ' + - 'range in request', () => s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: 'bytes=0-100', - }))).catch(err => { - checkNoError(err); - })); - - it('should copy a 0 byte object part from a source bucket to a ' + - 'different destination bucket and complete the MPU', () => { - const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - return s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: '', - })).then(() => s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, emptyFileETag); + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); assert(res.CopyPartResult.LastModified); - return s3.send(new CompleteMultipartUploadCommand({ + }), + ); + + it('should copy a part from a source bucket to a different ' + 'destination bucket and complete the MPU', () => + s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: emptyFileETag, PartNumber: 1 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); // AWS confirmed final ETag for MPU - assert.strictEqual(res.ETag,'"59adb24ef3cdbe0297f05b395827453f-1"'); - }); - })); - }); + assert.strictEqual(res.ETag, '"db77ebbae9e9f5a244a26b86193ad818-1"'); + }); + }), + ); - it('should copy a part using a range header from a source bucket ' + - 'to a different destination bucket and complete the MPU', () => { - const rangeETag = '"ac1be00f1f162e20d58099eec2ea1c70"'; - // AWS confirmed final ETag for MPU - const finalMpuETag = '"bff2a6af3adfd8e107a06de01d487176-1"'; - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - CopySourceRange: 'bytes=0-3', - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, rangeETag); - assert(res.CopyPartResult.LastModified); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: rangeETag, PartNumber: 1 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - assert.strictEqual(res.ETag, finalMpuETag); - return s3.send(new GetObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - })).then(async res => { - assert.strictEqual(res.ETag, finalMpuETag); - assert.strictEqual(res.ContentLength, 4); - const body = await res.Body.transformToString(); - assert.strictEqual(body, 'I am'); + it('should return InvalidArgument error given invalid range', () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: 'bad-range-parameter', + }), + ) + .catch(err => { + checkError(err, 'InvalidArgument'); + }), + )); + + it( + 'should return EntityTooLarge error if attempt to copy ' + + 'object larger than max and do not specify smaller ' + + 'range in request', + () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ), + ) + .catch(err => { + checkError(err, 'EntityTooLarge'); + }), + ); + + it( + 'should return EntityTooLarge error if attempt to copy ' + + 'object larger than max and specify too large ' + + 'range in request', + () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: `bytes=0-${oneHundredMBPlus11}`, + }), + ), + ) + .catch(err => { + checkError(err, 'EntityTooLarge'); + }), + ); + + it( + 'should succeed if attempt to copy ' + + 'object larger than max but specify acceptable ' + + 'range in request', + () => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: Buffer.alloc(oneHundredMBPlus11, 'packing'), + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: 'bytes=0-100', + }), + ), + ) + .catch(err => { + checkNoError(err); + }), + ); + + it( + 'should copy a 0 byte object part from a source bucket to a ' + + 'different destination bucket and complete the MPU', + () => { + const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; + return s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: '', + }), + ) + .then(() => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, emptyFileETag); + assert(res.CopyPartResult.LastModified); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: emptyFileETag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + // AWS confirmed final ETag for MPU + assert.strictEqual(res.ETag, '"59adb24ef3cdbe0297f05b395827453f-1"'); + }); + }), + ); + }, + ); + + it( + 'should copy a part using a range header from a source bucket ' + + 'to a different destination bucket and complete the MPU', + () => { + const rangeETag = '"ac1be00f1f162e20d58099eec2ea1c70"'; + // AWS confirmed final ETag for MPU + const finalMpuETag = '"bff2a6af3adfd8e107a06de01d487176-1"'; + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + CopySourceRange: 'bytes=0-3', + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, rangeETag); + assert(res.CopyPartResult.LastModified); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: rangeETag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + assert.strictEqual(res.ETag, finalMpuETag); + return s3 + .send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(async res => { + assert.strictEqual(res.ETag, finalMpuETag); + assert.strictEqual(res.ContentLength, 4); + const body = await res.Body.transformToString(); + assert.strictEqual(body, 'I am'); + }); + }); }); - }); - }); - }); + }, + ); describe('When copy source was put by MPU', () => { let sourceMpuId; const sourceMpuKey = 'sourceMpuKey'; // total hash for sourceMpuKey when MPU completed // (confirmed with AWS) - const totalMpuObjectHash = - '"9b0de95bd76728c778b9e25fd7ce2ef7"'; + const totalMpuObjectHash = '"9b0de95bd76728c778b9e25fd7ce2ef7"'; beforeEach(() => { const parts = []; @@ -301,379 +410,494 @@ describe('Object Part Copy', () => { const otherPartBuff = Buffer.alloc(5242880, 1); otherMd5HashPart.update(otherPartBuff); const otherPartHash = otherMd5HashPart.digest('hex'); - return s3.send(new CreateMultipartUploadCommand({ - Bucket: sourceBucketName, - Key: sourceMpuKey, - })).then(initiateRes => { - sourceMpuId = initiateRes.UploadId; - }).catch(err => { - process.stdout.write(`Error initiating MPU in MPU beforeEach: ${err}\n`); - throw err; - }).then(() => { - const partUploads = []; - // Concurrent uploads help trigger flakiness with "TimeoutError: - // socket hang up" due to a keep-alive race: the server closes - // an idle connection just as the client picks it from the pool. - const uploadWithRetry = (params, attempt = 0) => - s3.send(new UploadPartCommand(params)).catch(err => { - if (attempt < 3) { - process.stdout.write(`Retrying UploadPart ${params.PartNumber} ` - + `(attempt ${attempt + 1}/3): ${err}\n`); - return uploadWithRetry(params, attempt + 1); - } - throw err; - }); - for (let i = 1; i < 10; i++) { - const partBuffHere = i % 2 ? partBuff : otherPartBuff; - const partHashHere = i % 2 ? partHash : otherPartHash; - partUploads.push(uploadWithRetry({ + return s3 + .send( + new CreateMultipartUploadCommand({ Bucket: sourceBucketName, Key: sourceMpuKey, - PartNumber: i, - UploadId: sourceMpuId, - Body: partBuffHere, - })); - parts.push({ - ETag: partHashHere, - PartNumber: i, - }); - } - process.stdout.write('about to put parts\n'); - return Promise.all(partUploads); - }).catch(err => { - process.stdout.write(`Error putting parts in MPU beforeEach: ${err}\n`); - throw err; - }).then(() => { - process.stdout.write('completing mpu\n'); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: sourceBucketName, - Key: sourceMpuKey, - UploadId: sourceMpuId, - MultipartUpload: { - Parts: parts, - }, - })); - }).then(() => { - process.stdout.write('finished completing mpu\n'); - }).catch(err => { - process.stdout.write(`Error in MPU beforeEach: ${err}\n`); - throw err; - }); + }), + ) + .then(initiateRes => { + sourceMpuId = initiateRes.UploadId; + }) + .catch(err => { + process.stdout.write(`Error initiating MPU in MPU beforeEach: ${err}\n`); + throw err; + }) + .then(() => { + const partUploads = []; + // Concurrent uploads help trigger flakiness with "TimeoutError: + // socket hang up" due to a keep-alive race: the server closes + // an idle connection just as the client picks it from the pool. + const uploadWithRetry = (params, attempt = 0) => + s3.send(new UploadPartCommand(params)).catch(err => { + if (attempt < 3) { + process.stdout.write( + `Retrying UploadPart ${params.PartNumber} ` + + `(attempt ${attempt + 1}/3): ${err}\n`, + ); + return uploadWithRetry(params, attempt + 1); + } + throw err; + }); + for (let i = 1; i < 10; i++) { + const partBuffHere = i % 2 ? partBuff : otherPartBuff; + const partHashHere = i % 2 ? partHash : otherPartHash; + partUploads.push( + uploadWithRetry({ + Bucket: sourceBucketName, + Key: sourceMpuKey, + PartNumber: i, + UploadId: sourceMpuId, + Body: partBuffHere, + }), + ); + parts.push({ + ETag: partHashHere, + PartNumber: i, + }); + } + process.stdout.write('about to put parts\n'); + return Promise.all(partUploads); + }) + .catch(err => { + process.stdout.write(`Error putting parts in MPU beforeEach: ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('completing mpu\n'); + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: sourceBucketName, + Key: sourceMpuKey, + UploadId: sourceMpuId, + MultipartUpload: { + Parts: parts, + }, + }), + ); + }) + .then(() => { + process.stdout.write('finished completing mpu\n'); + }) + .catch(err => { + process.stdout.write(`Error in MPU beforeEach: ${err}\n`); + throw err; + }); }); - afterEach(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: sourceBucketName, - Key: sourceMpuKey, - UploadId: sourceMpuId, - })).catch(err => { - if (err.name !== 'NoSuchUpload' - && err.name !== 'NoSuchBucket') { - process.stdout.write(`Error in afterEach: ${err}\n`); - throw err; - } - })); + afterEach(() => + s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: sourceBucketName, + Key: sourceMpuKey, + UploadId: sourceMpuId, + }), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload' && err.name !== 'NoSuchBucket') { + process.stdout.write(`Error in afterEach: ${err}\n`); + throw err; + } + }), + ); - it('should copy a part from a source bucket to a different ' + - 'destination bucket', () => { + it('should copy a part from a source bucket to a different ' + 'destination bucket', () => { process.stdout.write('Entered first mpu test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, - totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }); - }); - - it('should copy two parts from a source bucket to a different ' + - 'destination bucket and complete the MPU', () => { - process.stdout.write('Putting first part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Putting second part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 2, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Completing MPU\n'); - return s3.send(new CompleteMultipartUploadCommand({ + return s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: totalMpuObjectHash, PartNumber: 1 }, - { ETag: totalMpuObjectHash, PartNumber: 2 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - // combined ETag returned by AWS (combination of part ETags - // with number of parts at the end) - assert.strictEqual(res.ETag, - '"5bba96810ff449d94aa8f5c5a859b0cb-2"'); - }).catch(err => { - checkNoError(err); - }); + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); }); - }); }); - it('should copy two parts with range headers from a source ' + - 'bucket to a different destination bucket and ' + - 'complete the MPU', () => { - process.stdout.write('Putting first part in MPU range test\n'); - const part1ETag = '"b1e0d096c8f0670c5367d131e392b84a"'; - const part2ETag = '"a2468d5c0ec2d4d5fc13b73beb63080a"'; - // combined ETag returned by AWS (combination of part ETags - // with number of parts at the end) - const finalCombinedETag = - '"e08ede4e8b942e18537cb2289f613ae3-2"'; - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - CopySourceRange: 'bytes=5242890-15242880', - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, part1ETag); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Putting second part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 2, - UploadId: uploadId, - CopySourceRange: 'bytes=15242891-30242991', - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, part2ETag); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Completing MPU\n'); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: part1ETag, PartNumber: 1 }, - { ETag: part2ETag, PartNumber: 2 }, - ], - }, - })).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - assert.strictEqual(res.ETag, finalCombinedETag); - }).then(() => { - process.stdout.write('Getting new object\n'); - return s3.send(new GetObjectCommand({ + it( + 'should copy two parts from a source bucket to a different ' + + 'destination bucket and complete the MPU', + () => { + process.stdout.write('Putting first part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - })).then(res => { - assert.strictEqual(res.ContentLength, 25000092); - assert.strictEqual(res.ETag, finalCombinedETag); - }) - .catch(err => { - checkNoError(err); - }); + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Putting second part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 2, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Completing MPU\n'); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { ETag: totalMpuObjectHash, PartNumber: 1 }, + { ETag: totalMpuObjectHash, PartNumber: 2 }, + ], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + // combined ETag returned by AWS (combination of part ETags + // with number of parts at the end) + assert.strictEqual(res.ETag, '"5bba96810ff449d94aa8f5c5a859b0cb-2"'); + }) + .catch(err => { + checkNoError(err); + }); + }); }); - }); - }); - }); + }, + ); + + it( + 'should copy two parts with range headers from a source ' + + 'bucket to a different destination bucket and ' + + 'complete the MPU', + () => { + process.stdout.write('Putting first part in MPU range test\n'); + const part1ETag = '"b1e0d096c8f0670c5367d131e392b84a"'; + const part2ETag = '"a2468d5c0ec2d4d5fc13b73beb63080a"'; + // combined ETag returned by AWS (combination of part ETags + // with number of parts at the end) + const finalCombinedETag = '"e08ede4e8b942e18537cb2289f613ae3-2"'; + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, + UploadId: uploadId, + CopySourceRange: 'bytes=5242890-15242880', + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, part1ETag); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Putting second part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 2, + UploadId: uploadId, + CopySourceRange: 'bytes=15242891-30242991', + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, part2ETag); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Completing MPU\n'); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { ETag: part1ETag, PartNumber: 1 }, + { ETag: part2ETag, PartNumber: 2 }, + ], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + assert.strictEqual(res.ETag, finalCombinedETag); + }) + .then(() => { + process.stdout.write('Getting new object\n'); + return s3 + .send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(res => { + assert.strictEqual(res.ContentLength, 25000092); + assert.strictEqual(res.ETag, finalCombinedETag); + }) + .catch(err => { + checkNoError(err); + }); + }); + }); + }); + }, + ); it('should overwrite an existing part by copying a part', () => { // AWS response etag for this completed MPU const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"'; process.stdout.write('Putting first part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceMpuKey}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); - assert(res.CopyPartResult.LastModified); - }).then(() => { - process.stdout.write('Overwriting first part in MPU test\n'); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId - }) - ).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - process.stdout.write('Completing MPU\n'); - return s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: destObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: etag, PartNumber: 1 }, - ], - }, - }) - ).then(res => { - assert.strictEqual(res.Bucket, destBucketName); - assert.strictEqual(res.Key, destObjName); - assert.strictEqual(res.ETag, finalObjETag); - }).then(() => { - process.stdout.write('Getting object put by MPU with ' + - 'overwrite part\n'); - return s3.send(new GetObjectCommand({ + return s3 + .send( + new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - })).then(res => { - assert.strictEqual(res.ETag, finalObjETag); - }).catch(err => { - checkNoError(err); - }); + CopySource: `${sourceBucketName}/${sourceMpuKey}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, totalMpuObjectHash); + assert(res.CopyPartResult.LastModified); + }) + .then(() => { + process.stdout.write('Overwriting first part in MPU test\n'); + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + process.stdout.write('Completing MPU\n'); + return s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: destObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .then(res => { + assert.strictEqual(res.Bucket, destBucketName); + assert.strictEqual(res.Key, destObjName); + assert.strictEqual(res.ETag, finalObjETag); + }) + .then(() => { + process.stdout.write('Getting object put by MPU with ' + 'overwrite part\n'); + return s3 + .send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + }), + ) + .then(res => { + assert.strictEqual(res.ETag, finalObjETag); + }) + .catch(err => { + checkNoError(err); + }); + }); + }); }); - }); }); - }); - it('should not corrupt object if overwriting an existing part by copying a part ' + - 'while the MPU is being completed', async () => { - const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"'; - process.stdout.write('Putting first part in MPU test\n'); - const randomDestObjName = `copycatobject${Math.floor(Math.random() * 100000)}`; + it( + 'should not corrupt object if overwriting an existing part by copying a part ' + + 'while the MPU is being completed', + async () => { + const finalObjETag = '"db77ebbae9e9f5a244a26b86193ad818-1"'; + process.stdout.write('Putting first part in MPU test\n'); + const randomDestObjName = `copycatobject${Math.floor(Math.random() * 100000)}`; - const initiateRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - })); + const initiateRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + }), + ); const uploadId = initiateRes.UploadId; - const res = await s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })); + const res = await s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ); assert.strictEqual(res.CopyPartResult.ETag, etag); assert(res.CopyPartResult.LastModified); - process.stdout.write( - 'Overwriting first part in MPU test and completing MPU at the same time\n', - ); + process.stdout.write('Overwriting first part in MPU test and completing MPU at the same time\n'); const [completeRes, uploadRes] = await Promise.all([ - s3.send(new CompleteMultipartUploadCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ ETag: etag, PartNumber: 1 }], - }, - })).catch(err => { - throw err; - }), - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })) - .catch(err => { - const completeMPUFinishedEarlier = - err.name === 'NoSuchKey' || err.name === 'NoSuchUpload'; - if (completeMPUFinishedEarlier) { - return Promise.resolve(null); - } - throw err; - }), - ], - ); + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: etag, PartNumber: 1 }], + }, + }), + ) + .catch(err => { + throw err; + }), + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { + const completeMPUFinishedEarlier = + err.name === 'NoSuchKey' || err.name === 'NoSuchUpload'; + if (completeMPUFinishedEarlier) { + return Promise.resolve(null); + } + throw err; + }), + ]); if (uploadRes !== null) { - assert.strictEqual(uploadRes.CopyPartResult.ETag, etag); - assert(uploadRes.CopyPartResult.LastModified); + assert.strictEqual(uploadRes.CopyPartResult.ETag, etag); + assert(uploadRes.CopyPartResult.LastModified); } assert.strictEqual(completeRes.Bucket, destBucketName); assert.strictEqual(completeRes.Key, randomDestObjName); assert.strictEqual(completeRes.ETag, finalObjETag); - process.stdout.write( - 'Getting object put by MPU with overwrite part\n', + process.stdout.write('Getting object put by MPU with overwrite part\n'); + const resGet = await s3.send( + new GetObjectCommand({ + Bucket: destBucketName, + Key: randomDestObjName, + }), ); - const resGet = await s3 - .send(new GetObjectCommand({ - Bucket: destBucketName, - Key: randomDestObjName, - })); assert.strictEqual(resGet.ETag, finalObjETag); - }); + }, + ); }); - it('should return an error if no such upload initiated', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: 'madeupuploadid444233232', - })).catch(err => { + it('should return an error if no such upload initiated', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: 'madeupuploadid444233232', + }), + ) + .catch(err => { checkError(err, 'NoSuchUpload'); })); - it('should return an error if attempt to copy from nonexistent bucket', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `nobucket453234/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => { + it('should return an error if attempt to copy from nonexistent bucket', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `nobucket453234/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'NoSuchBucket'); })); - it('should return an error if attempt to copy to nonexistent bucket', - () => s3.send(new UploadPartCopyCommand({ Bucket: 'nobucket453234', Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => { + it('should return an error if attempt to copy to nonexistent bucket', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: 'nobucket453234', + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'NoSuchBucket'); })); - it('should return an error if attempt to copy nonexistent object', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/nokey`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => { + it('should return an error if attempt to copy nonexistent object', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/nokey`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'NoSuchKey'); })); - it('should return an error if use invalid part number', - () => s3.send(new UploadPartCopyCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/nokey`, - PartNumber: 10001, - UploadId: uploadId, - })).catch(err => { + it('should return an error if use invalid part number', () => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/nokey`, + PartNumber: 10001, + UploadId: uploadId, + }), + ) + .catch(err => { checkError(err, 'InvalidArgument'); })); @@ -683,45 +907,51 @@ describe('Object Part Copy', () => { const archive = { archiveInfo: { archiveId: '97a71dfe-49c1-4cca-840a-69199e0b0322', - archiveVersion: 5577006791947779 + archiveVersion: 5577006791947779, }, }; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archive, err => { assert.ifError(err); - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(() => { - done(new Error('Expected failure but got success')); - }).catch(err => { - + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(() => { + done(new Error('Expected failure but got success')); + }) + .catch(err => { assert.strictEqual(err.$metadata.httpStatusCode, 403); done(); }); }); }); - it('should copy a part of an object when it\'s transitioning to cold', done => { + it("should copy a part of an object when it's transitioning to cold", done => { fakeMetadataTransition(sourceBucketName, sourceObjName, undefined, err => { assert.ifError(err); - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); - + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); }); @@ -731,24 +961,28 @@ describe('Object Part Copy', () => { restoreRequestedAt: new Date(0), restoreRequestedDays: 5, restoreCompletedAt: new Date(10), - restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)), + restoreWillExpireAt: new Date(10 + 5 * 24 * 60 * 60 * 1000), }; fakeMetadataArchive(sourceBucketName, sourceObjName, undefined, archiveCompleted, err => { assert.ifError(err); - s3.send(new UploadPartCopyCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: uploadId, - })).then(res => { - assert.strictEqual(res.CopyPartResult.ETag, etag); - assert(res.CopyPartResult.LastModified); - done(); - }).catch(err => { - checkNoError(err); - done(err); - }); + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(res => { + assert.strictEqual(res.CopyPartResult.ETag, etag); + assert(res.CopyPartResult.LastModified); + done(); + }) + .catch(err => { + checkNoError(err); + done(err); + }); }); }); }); @@ -760,82 +994,122 @@ describe('Object Part Copy', () => { beforeEach(() => { process.stdout.write('In other account before each\n'); - return otherAccountS3.send(new CreateBucketCommand({ Bucket: - otherAccountBucket })) - .catch(err => { - process.stdout.write('Error creating other account ' + - `bucket: ${err}\n`); - throw err; - }).then(() => { - process.stdout.write('Initiating other account MPU\n'); - return otherAccountS3.send(new CreateMultipartUploadCommand({ - Bucket: otherAccountBucket, - Key: otherAccountKey, - })); - }).then(initiateRes => { - otherAccountUploadId = initiateRes.UploadId; - }).catch(err => { - process.stdout.write('Error in other account ' + - `beforeEach: ${err}\n`); - throw err; - }); + return otherAccountS3 + .send(new CreateBucketCommand({ Bucket: otherAccountBucket })) + .catch(err => { + process.stdout.write('Error creating other account ' + `bucket: ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('Initiating other account MPU\n'); + return otherAccountS3.send( + new CreateMultipartUploadCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + }), + ); + }) + .then(initiateRes => { + otherAccountUploadId = initiateRes.UploadId; + }) + .catch(err => { + process.stdout.write('Error in other account ' + `beforeEach: ${err}\n`); + throw err; + }); }); - afterEach(() => otherAccountBucketUtility.empty(otherAccountBucket) - .then(() => otherAccountS3.send(new AbortMultipartUploadCommand({ - Bucket: otherAccountBucket, - Key: otherAccountKey, - UploadId: otherAccountUploadId, - }))) - .catch(err => { - if (err.name !== 'NoSuchUpload') { - process.stdout.write('Error in other account ' + - `afterEach: ${err}\n`); - throw err; - } - }).then(() => { - otherAccountBucketUtility.deleteOne(otherAccountBucket); - }) + afterEach(() => + otherAccountBucketUtility + .empty(otherAccountBucket) + .then(() => + otherAccountS3.send( + new AbortMultipartUploadCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + UploadId: otherAccountUploadId, + }), + ), + ) + .catch(err => { + if (err.name !== 'NoSuchUpload') { + process.stdout.write('Error in other account ' + `afterEach: ${err}\n`); + throw err; + } + }) + .then(() => { + otherAccountBucketUtility.deleteOne(otherAccountBucket); + }), ); - it('should not allow an account without read persmission on the ' + - 'source object to copy the object', () => otherAccountS3.send(new UploadPartCopyCommand( - { Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: otherAccountUploadId, - })).catch( - err => { - checkError(err, 'AccessDenied'); - })); - - it('should not allow an account without write persmission on the ' + - 'destination bucket to upload part copy the object', () => { - otherAccountS3.send(new PutObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, Body: '' })).then(() => otherAccountS3.send( - new UploadPartCopyCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${otherAccountBucket}/${otherAccountKey}`, - PartNumber: 1, - UploadId: uploadId, - })).catch(err => checkError(err, 'AccessDenied'))); - }); + it( + 'should not allow an account without read persmission on the ' + 'source object to copy the object', + () => + otherAccountS3 + .send( + new UploadPartCopyCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: otherAccountUploadId, + }), + ) + .catch(err => { + checkError(err, 'AccessDenied'); + }), + ); - it('should allow an account with read permission on the ' + - 'source object and write permission on the destination ' + - 'bucket to upload part copy the object', () => s3.send(new PutObjectAclCommand( - { Bucket: sourceBucketName, - Key: sourceObjName, ACL: 'public-read' })).then(() => otherAccountS3.send(new UploadPartCopyCommand( - { Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: `${sourceBucketName}/${sourceObjName}`, - PartNumber: 1, - UploadId: otherAccountUploadId, - })).catch(err => { - checkNoError(err); - } - ))); + it( + 'should not allow an account without write persmission on the ' + + 'destination bucket to upload part copy the object', + () => { + otherAccountS3 + .send(new PutObjectCommand({ Bucket: otherAccountBucket, Key: otherAccountKey, Body: '' })) + .then(() => + otherAccountS3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${otherAccountBucket}/${otherAccountKey}`, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .catch(err => checkError(err, 'AccessDenied')), + ); + }, + ); + + it( + 'should allow an account with read permission on the ' + + 'source object and write permission on the destination ' + + 'bucket to upload part copy the object', + () => + s3 + .send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'public-read', + }), + ) + .then(() => + otherAccountS3 + .send( + new UploadPartCopyCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: `${sourceBucketName}/${sourceObjName}`, + PartNumber: 1, + UploadId: otherAccountUploadId, + }), + ) + .catch(err => { + checkNoError(err); + }), + ), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js b/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js index 9621c84e8c..1ee4c1b722 100644 --- a/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/corsErrorHeaders.js @@ -8,8 +8,7 @@ const { const assert = require('assert'); const getConfig = require('../support/config'); -const { methodRequest, generateCorsParams } = - require('../../lib/utility/cors-util'); +const { methodRequest, generateCorsParams } = require('../../lib/utility/cors-util'); const config = getConfig('default', { signatureVersion: 'v4' }); const s3 = new S3Client({ ...config, forcePathStyle: true }); @@ -17,8 +16,7 @@ const s3 = new S3Client({ ...config, forcePathStyle: true }); const bucket = 'corserrorheadertest'; const objectKey = 'objectKey'; const allowedOrigin = 'http://www.allowed.test'; -const vary = 'Origin, Access-Control-Request-Headers, ' - + 'Access-Control-Request-Method'; +const vary = 'Origin, Access-Control-Request-Headers, ' + 'Access-Control-Request-Method'; const expectedCorsHeaders = { 'access-control-allow-origin': allowedOrigin, @@ -36,32 +34,19 @@ const corsParams = generateCorsParams(bucket, { // Raw unauthenticated requests - they always return 403. // Each spec describes (method, path, query) against the bucket. const unauthenticatedRequests = [ - { description: 'GET bucket (list objects)', - method: 'GET', query: null, objectKey: null }, - { description: 'HEAD bucket', - method: 'HEAD', query: null, objectKey: null }, - { description: 'DELETE bucket', - method: 'DELETE', query: null, objectKey: null }, - { description: 'GET bucket ACL', - method: 'GET', query: 'acl', objectKey: null }, - { description: 'GET bucket CORS', - method: 'GET', query: 'cors', objectKey: null }, - { description: 'GET bucket versioning', - method: 'GET', query: 'versioning', objectKey: null }, - { description: 'GET bucket website', - method: 'GET', query: 'website', objectKey: null }, - { description: 'GET bucket tagging', - method: 'GET', query: 'tagging', objectKey: null }, - { description: 'GET object', - method: 'GET', query: null, objectKey }, - { description: 'HEAD object', - method: 'HEAD', query: null, objectKey }, - { description: 'PUT object', - method: 'PUT', query: null, objectKey }, - { description: 'DELETE object', - method: 'DELETE', query: null, objectKey }, - { description: 'GET bucket uploads (list multipart uploads)', - method: 'GET', query: 'uploads', objectKey: null }, + { description: 'GET bucket (list objects)', method: 'GET', query: null, objectKey: null }, + { description: 'HEAD bucket', method: 'HEAD', query: null, objectKey: null }, + { description: 'DELETE bucket', method: 'DELETE', query: null, objectKey: null }, + { description: 'GET bucket ACL', method: 'GET', query: 'acl', objectKey: null }, + { description: 'GET bucket CORS', method: 'GET', query: 'cors', objectKey: null }, + { description: 'GET bucket versioning', method: 'GET', query: 'versioning', objectKey: null }, + { description: 'GET bucket website', method: 'GET', query: 'website', objectKey: null }, + { description: 'GET bucket tagging', method: 'GET', query: 'tagging', objectKey: null }, + { description: 'GET object', method: 'GET', query: null, objectKey }, + { description: 'HEAD object', method: 'HEAD', query: null, objectKey }, + { description: 'PUT object', method: 'PUT', query: null, objectKey }, + { description: 'DELETE object', method: 'DELETE', query: null, objectKey }, + { description: 'GET bucket uploads (list multipart uploads)', method: 'GET', query: 'uploads', objectKey: null }, // GET bucket policy and POST multi-delete are not covered here: the // first returns 405 (method rejected pre-auth), the second returns 400 // (missing XML body fails validation pre-auth). Neither reaches the @@ -80,27 +65,29 @@ describe('CORS headers on 403 responses when bucket has CORS configured', () => }); unauthenticatedRequests.forEach(spec => { - it(`returns CORS headers on 403 for ${spec.description} ` - + 'when Origin matches a rule', done => { - methodRequest({ - method: spec.method, - bucket, - objectKey: spec.objectKey, - query: spec.query, - headers: { origin: allowedOrigin }, - // Use numeric status: HEAD responses have no body, and some - // endpoints (bucket policy, multi-delete) can fail with a - // non-AccessDenied body before auth even runs. We only care - // about the 403 status and the CORS headers here. - code: 403, - headersResponse: expectedCorsHeaders, - }, done); + it(`returns CORS headers on 403 for ${spec.description} ` + 'when Origin matches a rule', done => { + methodRequest( + { + method: spec.method, + bucket, + objectKey: spec.objectKey, + query: spec.query, + headers: { origin: allowedOrigin }, + // Use numeric status: HEAD responses have no body, and some + // endpoints (bucket policy, multi-delete) can fail with a + // non-AccessDenied body before auth even runs. We only care + // about the 403 status and the CORS headers here. + code: 403, + headersResponse: expectedCorsHeaders, + }, + done, + ); }); }); - it('omits CORS headers on 403 when Origin does not match any rule', - done => { - methodRequest({ + it('omits CORS headers on 403 when Origin does not match any rule', done => { + methodRequest( + { method: 'GET', bucket, query: null, @@ -109,20 +96,24 @@ describe('CORS headers on 403 responses when bucket has CORS configured', () => code: 403, // headersResponse unset -> cors-util asserts CORS headers // are NOT present. - }, done); - }); + }, + done, + ); + }); - it('omits CORS headers on 403 when no Origin header is sent', - done => { - methodRequest({ + it('omits CORS headers on 403 when no Origin header is sent', done => { + methodRequest( + { method: 'GET', bucket, query: null, objectKey: null, headers: {}, code: 403, - }, done); - }); + }, + done, + ); + }); }); describe('CORS headers on 200 responses (regression guard)', () => { @@ -135,26 +126,30 @@ describe('CORS headers on 200 responses (regression guard)', () => { await s3.send(new DeleteBucketCommand({ Bucket: bucket })); }); - it('returns CORS headers on a successful list objects (200)', - async () => { - const command = new ListObjectsCommand({ Bucket: bucket }); - // Inject Origin on the outgoing request and capture the raw - // response headers via the deserialize step. - command.middlewareStack.add(next => async args => { + it('returns CORS headers on a successful list objects (200)', async () => { + const command = new ListObjectsCommand({ Bucket: bucket }); + // Inject Origin on the outgoing request and capture the raw + // response headers via the deserialize step. + command.middlewareStack.add( + next => async args => { const headers = args.request && args.request.headers; if (headers) { headers.origin = allowedOrigin; } return next(args); - }, { step: 'build' }); - let responseHeaders; - command.middlewareStack.add(next => async args => { + }, + { step: 'build' }, + ); + let responseHeaders; + command.middlewareStack.add( + next => async args => { const result = await next(args); responseHeaders = result.response && result.response.headers; return result; - }, { step: 'deserialize' }); - await s3.send(command); - assert.strictEqual(responseHeaders['access-control-allow-origin'], - allowedOrigin); - }); + }, + { step: 'deserialize' }, + ); + await s3.send(command); + assert.strictEqual(responseHeaders['access-control-allow-origin'], allowedOrigin); + }); }); diff --git a/tests/functional/aws-node-sdk/test/object/corsHeaders.js b/tests/functional/aws-node-sdk/test/object/corsHeaders.js index 45d0e1a209..4da1f2dc22 100644 --- a/tests/functional/aws-node-sdk/test/object/corsHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/corsHeaders.js @@ -1,35 +1,37 @@ -const { S3Client, - ListObjectsCommand, - GetBucketAclCommand, - GetBucketCorsCommand, - GetBucketVersioningCommand, - GetBucketLocationCommand, - GetBucketWebsiteCommand, - ListMultipartUploadsCommand, - GetObjectCommand, - GetObjectAclCommand, - ListPartsCommand, - HeadBucketCommand, - HeadObjectCommand, - CreateBucketCommand, - PutBucketAclCommand, - PutBucketVersioningCommand, - PutBucketWebsiteCommand, - PutBucketCorsCommand, - PutObjectCommand, - PutObjectAclCommand, - CopyObjectCommand, - UploadPartCommand, - UploadPartCopyCommand, - CreateMultipartUploadCommand, - CompleteMultipartUploadCommand, - DeleteObjectsCommand, - DeleteBucketCommand, - DeleteBucketWebsiteCommand, - DeleteBucketCorsCommand, - DeleteObjectCommand, - AbortMultipartUploadCommand, - ListBucketsCommand } = require('@aws-sdk/client-s3'); +const { + S3Client, + ListObjectsCommand, + GetBucketAclCommand, + GetBucketCorsCommand, + GetBucketVersioningCommand, + GetBucketLocationCommand, + GetBucketWebsiteCommand, + ListMultipartUploadsCommand, + GetObjectCommand, + GetObjectAclCommand, + ListPartsCommand, + HeadBucketCommand, + HeadObjectCommand, + CreateBucketCommand, + PutBucketAclCommand, + PutBucketVersioningCommand, + PutBucketWebsiteCommand, + PutBucketCorsCommand, + PutObjectCommand, + PutObjectAclCommand, + CopyObjectCommand, + UploadPartCommand, + UploadPartCopyCommand, + CreateMultipartUploadCommand, + CompleteMultipartUploadCommand, + DeleteObjectsCommand, + DeleteBucketCommand, + DeleteBucketWebsiteCommand, + DeleteBucketCorsCommand, + DeleteObjectCommand, + AbortMultipartUploadCommand, + ListBucketsCommand, +} = require('@aws-sdk/client-s3'); const { promisify } = require('util'); const assert = require('assert'); @@ -48,8 +50,7 @@ const bucket = 'bucketcorsheadertest'; const objectKey = 'objectKeyName'; const allowedOrigin = 'http://www.allowedwebsite.com'; const notAllowedOrigin = 'http://www.notallowedwebsite.com'; -const vary = 'Origin, Access-Control-Request-Headers, ' + - 'Access-Control-Request-Method'; +const vary = 'Origin, Access-Control-Request-Headers, ' + 'Access-Control-Request-Method'; const defaultOptions = { allowedMethods: ['GET'], allowedOrigins: [allowedOrigin], @@ -152,10 +153,12 @@ const apiMethods = [ params: { Bucket: bucket, CORSConfiguration: { - CORSRules: [{ - AllowedOrigins: [allowedOrigin], - AllowedMethods: ['PUT'], - }], + CORSRules: [ + { + AllowedOrigins: [allowedOrigin], + AllowedMethods: ['PUT'], + }, + ], }, }, }, @@ -219,9 +222,7 @@ const apiMethods = [ params: { Bucket: bucket, Delete: { - Objects: [ - { Key: objectKey }, - ], + Objects: [{ Key: objectKey }], }, }, }, @@ -270,10 +271,12 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { }); } else { // if no expectedHeaders provided, should not have these headers in the response - ['access-control-allow-origin', - 'access-control-allow-methods', - 'access-control-allow-credentials', - 'vary'].forEach(key => { + [ + 'access-control-allow-origin', + 'access-control-allow-methods', + 'access-control-allow-credentials', + 'vary', + ].forEach(key => { assert.strictEqual(resHeaders[key], undefined, `Error: ${key} should not have value`); }); } @@ -319,8 +322,8 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { { step: 'finalizeRequest', name: 'captureHeaders', - priority: 'high' - } + priority: 'high', + }, ); try { @@ -330,15 +333,16 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { // Clean up multipart upload if needed (equivalent to the original cleanup logic) if (response.UploadId) { - await testS3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - UploadId: response.UploadId - })); + await testS3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + UploadId: response.UploadId, + }), + ); } _runAssertions(capturedHeaders); - } catch { // CORS headers should still be sent in case of errors as long as // request matches CORS configuration @@ -348,15 +352,17 @@ async function _checkHeaders(action, params, origin, expectedHeaders) { describe('Cross Origin Resource Sharing requests', () => { beforeEach(done => { - s3.send(new CreateBucketCommand({ - Bucket: bucket, - ACL: 'public-read-write' - })) - .then(() => _waitForAWS(done)) - .catch(err => { - process.stdout.write(`Error in beforeEach: ${err}\n`); - _waitForAWS(done, err); - }); + s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ACL: 'public-read-write', + }), + ) + .then(() => _waitForAWS(done)) + .catch(err => { + process.stdout.write(`Error in beforeEach: ${err}\n`); + _waitForAWS(done, err); + }); }); afterEach(done => { @@ -372,29 +378,28 @@ describe('Cross Origin Resource Sharing requests', () => { }); describe('on non-existing bucket', () => { - it('should not respond to request with CORS headers, even if request was sent with Origin header', - async () => { + it('should not respond to request with CORS headers, even if request was sent with Origin header', async () => { await _checkHeaders(ListObjectsCommand, { Bucket: 'nonexistingbucket' }, allowedOrigin, null); }); }); describe('on bucket without CORS configuration', () => { - it('should not respond to request with CORS headers,' + - ' even if request was sent with Origin header', async () => { - await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); - }); + it( + 'should not respond to request with CORS headers,' + ' even if request was sent with Origin header', + async () => { + await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); + }, + ); }); - describe('on bucket with CORS configuration: ' + - 'allow one origin and all methods', () => { + describe('on bucket with CORS configuration: ' + 'allow one origin and all methods', () => { const corsParams = generateCorsParams(bucket, { allowedMethods: ['GET', 'PUT', 'HEAD', 'POST', 'DELETE'], allowedOrigins: [allowedOrigin], }); const expectedHeaders = { 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': corsParams.CORSConfiguration - .CORSRules[0].AllowedMethods.join(', '), + 'access-control-allow-methods': corsParams.CORSConfiguration.CORSRules[0].AllowedMethods.join(', '), 'access-control-allow-credentials': 'true', vary, }; @@ -405,8 +410,7 @@ describe('Cross Origin Resource Sharing requests', () => { afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { - if (err && err.name !== 'NoSuchKey' && - err.name !== 'NoSuchBucket') { + if (err && err.name !== 'NoSuchKey' && err.name !== 'NoSuchBucket') { process.stdout.write(`Unexpected err in afterEach: ${err}`); return done(err); } @@ -415,16 +419,22 @@ describe('Cross Origin Resource Sharing requests', () => { }); describe('when request Origin/method match CORS configuration', () => { - it('should not respond with CORS headers to GET service (list buckets), ' + - 'even if Origin/method match CORS rule', async () => { - await _checkHeaders(ListBucketsCommand, {}, allowedOrigin, null); - }); - - it('should not respond with CORS headers after deleting bucket, ' + - 'even if Origin/method match CORS rule', async () => { - await s3.send(new DeleteBucketCommand({ Bucket: bucket })); - await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); - }); + it( + 'should not respond with CORS headers to GET service (list buckets), ' + + 'even if Origin/method match CORS rule', + async () => { + await _checkHeaders(ListBucketsCommand, {}, allowedOrigin, null); + }, + ); + + it( + 'should not respond with CORS headers after deleting bucket, ' + + 'even if Origin/method match CORS rule', + async () => { + await s3.send(new DeleteBucketCommand({ Bucket: bucket })); + await _checkHeaders(ListObjectsCommand, { Bucket: bucket }, allowedOrigin, null); + }, + ); apiMethods.forEach(method => { it(`should respond to ${method.description} with CORS headers (access-control-allow-origin, @@ -462,10 +472,8 @@ describe('Cross Origin Resource Sharing requests', () => { }); }); - describe('on bucket with CORS configuration and website configuration', - () => { - const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; + describe('on bucket with CORS configuration and website configuration', () => { + const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; const corsParams = generateCorsParams(bucket, { allowedMethods: ['GET', 'HEAD'], allowedOrigins: [allowedOrigin], @@ -485,46 +493,67 @@ describe('Cross Origin Resource Sharing requests', () => { await s3.send(new CreateBucketCommand({ Bucket: bucket, ACL: 'public-read' })); await s3.send(new PutBucketCorsCommand(corsParams)); await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: 'test content' })); + await s3.send( + new PutObjectCommand({ Bucket: bucket, Key: 'index.html', ACL: 'public-read', Body: 'test content' }), + ); }); afterEach(done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: 'index.html' - })) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - .then(() => _waitForAWS(done)) - .catch(err => { - process.stdout.write(`Error in afterEach: ${err}\n`); - _waitForAWS(done, err); - }); + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: 'index.html', + }), + ) + .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) + .then(() => _waitForAWS(done)) + .catch(err => { + process.stdout.write(`Error in afterEach: ${err}\n`); + _waitForAWS(done, err); + }); }); it('should respond with CORS headers at website endpoint (GET)', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'GET', bucket, - headers, headersResponse, code: 200, isWebsite: true }); + await methodRequestPromise({ method: 'GET', bucket, headers, headersResponse, code: 200, isWebsite: true }); }); it('should respond with CORS headers at website endpoint (GET) even in case of error', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'GET', bucket, objectKey: 'test', - headers, headersResponse, code: 404, isWebsite: true }); + await methodRequestPromise({ + method: 'GET', + bucket, + objectKey: 'test', + headers, + headersResponse, + code: 404, + isWebsite: true, + }); }); it('should respond with CORS headers at website endpoint (GET) even in case of redirect', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'GET', bucket, objectKey: 'redirect', - headers, headersResponse, code: 301, isWebsite: true }); + await methodRequestPromise({ + method: 'GET', + bucket, + objectKey: 'redirect', + headers, + headersResponse, + code: 301, + isWebsite: true, + }); }); it('should respond with CORS headers at website endpoint (HEAD)', async () => { const headers = { Origin: allowedOrigin }; - await methodRequestPromise({ method: 'HEAD', bucket, headers, headersResponse, - code: 200, isWebsite: true }); + await methodRequestPromise({ + method: 'HEAD', + bucket, + headers, + headersResponse, + code: 200, + isWebsite: true, + }); }); }); @@ -548,27 +577,38 @@ describe('Cross Origin Resource Sharing requests', () => { await s3.send(new PutBucketCorsCommand(corsParams)); }); - it('should not return access-control-allow-headers response header ' + - 'even if request matches CORS rule and other access-control headers are returned', async () => { - const headers = { - 'Origin': allowedOrigin, - 'Content-Type': 'testvalue', - }; - const headersOmitted = ['access-control-allow-headers']; - await methodRequestPromise({ method: 'GET', bucket, headers, headersResponse, - headersOmitted, code: 200 }); - }); - - it('Request with matching Origin/method but additional headers that violate CORS rule:\n\t should still ' + - 'respond with access-control headers (headers are only checked in preflight requests)', async () => { - const headers = { - Origin: allowedOrigin, - Test: 'test', - Expires: 86400, - }; - await methodRequestPromise({ method: 'GET', bucket, headers, - headersResponse, code: 200 }); - }); + it( + 'should not return access-control-allow-headers response header ' + + 'even if request matches CORS rule and other access-control headers are returned', + async () => { + const headers = { + Origin: allowedOrigin, + 'Content-Type': 'testvalue', + }; + const headersOmitted = ['access-control-allow-headers']; + await methodRequestPromise({ + method: 'GET', + bucket, + headers, + headersResponse, + headersOmitted, + code: 200, + }); + }, + ); + + it( + 'Request with matching Origin/method but additional headers that violate CORS rule:\n\t should still ' + + 'respond with access-control headers (headers are only checked in preflight requests)', + async () => { + const headers = { + Origin: allowedOrigin, + Test: 'test', + Expires: 86400, + }; + await methodRequestPromise({ method: 'GET', bucket, headers, headersResponse, code: 200 }); + }, + ); }); [ diff --git a/tests/functional/aws-node-sdk/test/object/corsPreflight.js b/tests/functional/aws-node-sdk/test/object/corsPreflight.js index 41109f6d3c..7a79179c8f 100644 --- a/tests/functional/aws-node-sdk/test/object/corsPreflight.js +++ b/tests/functional/aws-node-sdk/test/object/corsPreflight.js @@ -17,14 +17,9 @@ const s3 = new S3Client(config); const bucket = 'bucketcorstester'; const methods = ['PUT', 'POST', 'DELETE', 'GET']; -const originsWithWildcards = [ - '*.allowedorigin.com', - 'http://*.allowedorigin.com', - 'http://www.allowedorigin.*', -]; +const originsWithWildcards = ['*.allowedorigin.com', 'http://*.allowedorigin.com', 'http://www.allowedorigin.*']; const allowedOrigin = 'http://www.allowedwebsite.com'; -const vary = 'Origin, Access-Control-Request-Headers, ' + - 'Access-Control-Request-Method'; +const vary = 'Origin, Access-Control-Request-Headers, ' + 'Access-Control-Request-Method'; // AWS seems to take a bit long so sometimes by the time we send the request // the bucket has not yet been created or the bucket has been deleted. @@ -41,21 +36,17 @@ describe('Preflight CORS request on non-existing bucket', () => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'GET', bucket, headers, code: 'NoSuchBucket', - headersResponse: null }, done); + methodRequest({ method: 'GET', bucket, headers, code: 'NoSuchBucket', headersResponse: null }, done); }); it('should return BadRequest for OPTIONS request without origin', done => { const headers = {}; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', - headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', headersResponse: null }, done); }); - it('should return BadRequest for OPTIONS request without ' + - 'Access-Control-Request-Method', done => { + it('should return BadRequest for OPTIONS request without ' + 'Access-Control-Request-Method', done => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', - headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'BadRequest', headersResponse: null }, done); }); }); @@ -71,45 +62,34 @@ describe('Preflight CORS request with existing bucket', () => { .catch(err => _waitForAWS(done, err)); }); - it('should allow GET on bucket without cors configuration even if ' + - 'Origin header sent', done => { + it('should allow GET on bucket without cors configuration even if ' + 'Origin header sent', done => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'GET', bucket, headers, code: 200, - headersResponse: null }, done); + methodRequest({ method: 'GET', bucket, headers, code: 200, headersResponse: null }, done); }); - it('should allow HEAD on bucket without cors configuration even if ' + - 'Origin header sent', done => { + it('should allow HEAD on bucket without cors configuration even if ' + 'Origin header sent', done => { const headers = { Origin: allowedOrigin, }; - methodRequest({ method: 'HEAD', bucket, headers, code: 200, - headersResponse: null }, done); + methodRequest({ method: 'HEAD', bucket, headers, code: 200, headersResponse: null }, done); }); - it('should respond AccessForbidden for OPTIONS request on bucket without ' + - 'CORSConfiguration', done => { + it('should respond AccessForbidden for OPTIONS request on bucket without ' + 'CORSConfiguration', done => { const headers = { - 'Origin': allowedOrigin, + Origin: allowedOrigin, 'Access-Control-Request-Method': 'GET', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, done); }); - describe('allow PUT, POST, DELETE, GET methods and allow only ' + - 'one origin', () => { + describe('allow PUT, POST, DELETE, GET methods and allow only ' + 'one origin', () => { const corsParams = { Bucket: bucket, CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'PUT', 'POST', 'DELETE', 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - ], + AllowedMethods: ['PUT', 'POST', 'DELETE', 'GET'], + AllowedOrigins: [allowedOrigin], }, ], }, @@ -127,43 +107,46 @@ describe('Preflight CORS request with existing bucket', () => { }); methods.forEach(method => { - it('should respond with 200 and access control headers to ' + - 'OPTIONS request from allowed origin and allowed method ' + - `"${method}"`, done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': method, - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'PUT, POST, DELETE, GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - }); - it('should respond AccessForbidden to OPTIONS request from ' + - 'not allowed origin', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'Content-Type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + it( + 'should respond with 200 and access control headers to ' + + 'OPTIONS request from allowed origin and allowed method ' + + `"${method}"`, + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': method, + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'PUT, POST, DELETE, GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); - it('should respond AccessForbidden to OPTIONS request with ' + - 'not allowed Access-Control-Request-Headers', done => { + it('should respond AccessForbidden to OPTIONS request from ' + 'not allowed origin', done => { const headers = { - 'Origin': 'http://www.forbiddenwebsite.com', + Origin: allowedOrigin, 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'Content-Type', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + methodRequest({ method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, done); }); + it( + 'should respond AccessForbidden to OPTIONS request with ' + 'not allowed Access-Control-Request-Headers', + done => { + const headers = { + Origin: 'http://www.forbiddenwebsite.com', + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); describe('CORS allows method GET and allows one origin', () => { @@ -172,12 +155,8 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - ], + AllowedMethods: ['GET'], + AllowedOrigins: [allowedOrigin], }, ], }, @@ -194,52 +173,66 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin and method "GET"', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond AccessForbidden to OPTIONS request with allowed ' + - 'method but not from allowed origin', done => { - const headers = { - 'Origin': 'http://www.forbiddenwebsite.com', - 'Access-Control-Request-Method': 'GET', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); - it('should respond AccessForbidden to OPTIONS request from allowed ' + - 'origin and method but with not allowed Access-Control-Request-Headers', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'Content-Type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); - ['PUT', 'POST', 'DELETE'].forEach(method => { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed origin but not allowed method "${method}"`, done => { + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin and method "GET"', + done => { const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': method, + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request with allowed ' + 'method but not from allowed origin', + done => { + const headers = { + Origin: 'http://www.forbiddenwebsite.com', + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request from allowed ' + + 'origin and method but with not allowed Access-Control-Request-Headers', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'Content-Type', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); + ['PUT', 'POST', 'DELETE'].forEach(method => { + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed origin but not allowed method "${method}"`, + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': method, + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); }); @@ -255,8 +248,7 @@ describe('Preflight CORS request with existing bucket', () => { ], }, }; - describe(`CORS allows method "${allowedMethod}" and allows all origins`, - () => { + describe(`CORS allows method "${allowedMethod}" and allows all origins`, () => { beforeEach(done => { s3.send(new PutBucketCorsCommand(corsParams)) .then(() => done()) @@ -269,45 +261,56 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should respond with 200 and access control headers to ' + - `OPTIONS request from allowed origin and method "${allowedMethod}"`, - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': allowedMethod, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': allowedMethod, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond AccessForbidden to OPTIONS request from ' + - 'allowed origin and method but with not allowed Access-Control-' + - 'Request-Headers', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': allowedMethod, - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'Content-Type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); - methods.filter(method => method !== allowedMethod) - .forEach(method => { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed origin but not allowed method "${method}"`, done => { + it( + 'should respond with 200 and access control headers to ' + + `OPTIONS request from allowed origin and method "${allowedMethod}"`, + done => { const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': method, + Origin: allowedOrigin, + 'Access-Control-Request-Method': allowedMethod, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': allowedMethod, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request from ' + + 'allowed origin and method but with not allowed Access-Control-' + + 'Request-Headers', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': allowedMethod, + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'Content-Type', }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); + methods + .filter(method => method !== allowedMethod) + .forEach(method => { + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed origin but not allowed method "${method}"`, + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': method, + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); - }); }); }); @@ -339,54 +342,61 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - [originWithoutWildcard, originReplaceWildcard] - .forEach(acceptableOrigin => { - it('should return 200 and CORS header to OPTIONS request ' + - `from allowed method and origin "${acceptableOrigin}"`, - done => { - const headers = { - 'Origin': acceptableOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': acceptableOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 200, headersResponse }, done); - }); + [originWithoutWildcard, originReplaceWildcard].forEach(acceptableOrigin => { + it( + 'should return 200 and CORS header to OPTIONS request ' + + `from allowed method and origin "${acceptableOrigin}"`, + done => { + const headers = { + Origin: acceptableOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': acceptableOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); if (!origin.endsWith('*')) { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed method and origin "${originWithoutWildcard}test"`, - done => { - const headers = { - 'Origin': `${originWithoutWildcard}test`, - 'Access-Control-Request-Method': 'GET', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed method and origin "${originWithoutWildcard}test"`, + done => { + const headers = { + Origin: `${originWithoutWildcard}test`, + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); } if (!origin.startsWith('*')) { - it('should respond AccessForbidden to OPTIONS request from ' + - `allowed method and origin "test${originWithoutWildcard}"`, - done => { - const headers = { - 'Origin': `test${originWithoutWildcard}`, - 'Access-Control-Request-Method': 'GET', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + it( + 'should respond AccessForbidden to OPTIONS request from ' + + `allowed method and origin "test${originWithoutWildcard}"`, + done => { + const headers = { + Origin: `test${originWithoutWildcard}`, + 'Access-Control-Request-Method': 'GET', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); } }); }); - describe('CORS response access-control-allow-origin header value', - () => { + describe('CORS response access-control-allow-origin header value', () => { const anotherOrigin = 'http://www.anotherorigin.com'; const originContainingWildcard = 'http://www.originwith*.com'; const corsParams = { @@ -394,21 +404,12 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - originContainingWildcard, - ], + AllowedMethods: ['GET'], + AllowedOrigins: [allowedOrigin, originContainingWildcard], }, { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], }, ], }, @@ -425,72 +426,71 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('if OPTIONS request matches rule with multiple origins, response ' + - 'access-control-request-origin header value should be request Origin ' + - '(not list of AllowedOrigins)', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('if OPTIONS request matches rule with origin containing wildcard, ' + - 'response access-control-request-origin header value should be ' + - 'request Origin (not value containing wildcard)', done => { - const requestOrigin = originContainingWildcard.replace('*', 'test'); - const headers = { - 'Origin': requestOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': requestOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('if OPTIONS request matches rule that allows all origins, ' + - 'e.g. "*", response access-control-request-origin header should ' + - 'return "*"', done => { - const headers = { - 'Origin': anotherOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'if OPTIONS request matches rule with multiple origins, response ' + + 'access-control-request-origin header value should be request Origin ' + + '(not list of AllowedOrigins)', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'if OPTIONS request matches rule with origin containing wildcard, ' + + 'response access-control-request-origin header value should be ' + + 'request Origin (not value containing wildcard)', + done => { + const requestOrigin = originContainingWildcard.replace('*', 'test'); + const headers = { + Origin: requestOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': requestOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'if OPTIONS request matches rule that allows all origins, ' + + 'e.g. "*", response access-control-request-origin header should ' + + 'return "*"', + done => { + const headers = { + Origin: anotherOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); - describe('CORS allows method GET, allows all origins and allows ' + - 'header Content-Type', () => { + describe('CORS allows method GET, allows all origins and allows ' + 'header Content-Type', () => { const corsParams = { Bucket: bucket, CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - AllowedHeaders: [ - 'content-type', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + AllowedHeaders: ['content-type'], }, ], }, @@ -507,80 +507,74 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin and method, even without request ' + - 'Access-Control-Request-Headers header value', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin and method with Access-Control-' + - 'Request-Headers \'Content-Type\'', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'content-type', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': 'content-type', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should respond AccessForbidden to OPTIONS request from allowed ' + - 'origin and method but not allowed Access-Control-Request-Headers ' + - 'in addition to Content-Type', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': 'Origin, Accept, ' + - 'content-type', - }; - methodRequest({ method: 'OPTIONS', bucket, headers, - code: 'AccessForbidden', headersResponse: null }, done); - }); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin and method, even without request ' + + 'Access-Control-Request-Headers header value', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin and method with Access-Control-' + + "Request-Headers 'Content-Type'", + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'content-type', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': 'content-type', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond AccessForbidden to OPTIONS request from allowed ' + + 'origin and method but not allowed Access-Control-Request-Headers ' + + 'in addition to Content-Type', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': 'Origin, Accept, ' + 'content-type', + }; + methodRequest( + { method: 'OPTIONS', bucket, headers, code: 'AccessForbidden', headersResponse: null }, + done, + ); + }, + ); }); - describe('CORS response Access-Control-Allow-Headers header value', - () => { + describe('CORS response Access-Control-Allow-Headers header value', () => { const corsParams = { Bucket: bucket, CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - AllowedHeaders: [ - 'Content-Type', 'amz-*', 'Expires', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + AllowedHeaders: ['Content-Type', 'amz-*', 'Expires'], }, { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - AllowedHeaders: [ - '*', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + AllowedHeaders: ['*'], }, ], }, @@ -597,82 +591,86 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('should return request access-control-request-headers value, ' + - 'not list of AllowedHeaders from rule or corresponding AllowedHeader ' + - 'value containing wildcard', - done => { - const requestHeaderValue = 'amz-meta-header-test, content-type'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': requestHeaderValue, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should return lowercase version of request Access-Control-' + - 'Request-Method header value if it contains any upper-case values', - done => { - const requestHeaderValue = 'Content-Type'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': - requestHeaderValue.toLowerCase(), - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should remove empty comma-separated values derived from request ' + - 'Access-Control-Request-Method header and separate values with ' + - 'spaces when responding with Access-Control-Allow-Headers header', - done => { - const requestHeaderValue = 'content-type,,expires'; - const expectedValue = 'content-type, expires'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': expectedValue, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); - it('should return request Access-Control-Request-Headers value ' + - 'even if rule allows all headers (e.g. "*"), unlike access-control-' + - 'allow-origin value', done => { - const requestHeaderValue = 'puppies'; - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Access-Control-Request-Headers': requestHeaderValue, - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-allow-headers': requestHeaderValue, - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'should return request access-control-request-headers value, ' + + 'not list of AllowedHeaders from rule or corresponding AllowedHeader ' + + 'value containing wildcard', + done => { + const requestHeaderValue = 'amz-meta-header-test, content-type'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': requestHeaderValue, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should return lowercase version of request Access-Control-' + + 'Request-Method header value if it contains any upper-case values', + done => { + const requestHeaderValue = 'Content-Type'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': requestHeaderValue.toLowerCase(), + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should remove empty comma-separated values derived from request ' + + 'Access-Control-Request-Method header and separate values with ' + + 'spaces when responding with Access-Control-Allow-Headers header', + done => { + const requestHeaderValue = 'content-type,,expires'; + const expectedValue = 'content-type, expires'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': expectedValue, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should return request Access-Control-Request-Headers value ' + + 'even if rule allows all headers (e.g. "*"), unlike access-control-' + + 'allow-origin value', + done => { + const requestHeaderValue = 'puppies'; + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + 'Access-Control-Request-Headers': requestHeaderValue, + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-allow-headers': requestHeaderValue, + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); describe('CORS and OPTIONS request with object keys', () => { @@ -681,12 +679,8 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - allowedOrigin, - ], + AllowedMethods: ['GET'], + AllowedOrigins: [allowedOrigin], }, ], }, @@ -701,10 +695,14 @@ describe('Preflight CORS request with existing bucket', () => { afterEach(done => { s3.send(new DeleteBucketCorsCommand({ Bucket: bucket })) - .then(() => s3.send(new DeleteObjectCommand({ - Key: objectKey, - Bucket: bucket, - }))) + .then(() => + s3.send( + new DeleteObjectCommand({ + Key: objectKey, + Bucket: bucket, + }), + ), + ) .then(() => done()) .catch(err => { process.stdout.write(`err in afterEach ${err}`); @@ -712,38 +710,44 @@ describe('Preflight CORS request with existing bucket', () => { }); }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin, allowed method and existing object key', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', objectKey, bucket, headers, - code: 200, headersResponse }, done); - }); - it('should respond with 200 and access control headers to OPTIONS ' + - 'request from allowed origin, allowed method, even with non-existing ' + - 'object key', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': allowedOrigin, - 'access-control-allow-methods': 'GET', - 'access-control-allow-credentials': 'true', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, objectKey: - 'anotherObjectKey', headers, code: 200, headersResponse }, done); - }); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin, allowed method and existing object key', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest({ method: 'OPTIONS', objectKey, bucket, headers, code: 200, headersResponse }, done); + }, + ); + it( + 'should respond with 200 and access control headers to OPTIONS ' + + 'request from allowed origin, allowed method, even with non-existing ' + + 'object key', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': allowedOrigin, + 'access-control-allow-methods': 'GET', + 'access-control-allow-credentials': 'true', + vary, + }; + methodRequest( + { method: 'OPTIONS', bucket, objectKey: 'anotherObjectKey', headers, code: 200, headersResponse }, + done, + ); + }, + ); }); describe('CORS and OPTIONS request', () => { @@ -770,37 +774,41 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('with fake auth credentials: should respond with 200 and access ' + - 'control headers even if request has fake auth credentials', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Authorization': 'AWS fakeKey:fakesignature', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'with fake auth credentials: should respond with 200 and access ' + + 'control headers even if request has fake auth credentials', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + Authorization: 'AWS fakeKey:fakesignature', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); - it('with cookies: should send identical response as to request ' + - 'without cookies (200 and access control headers)', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - 'Cookie': 'testcookie=1', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'with cookies: should send identical response as to request ' + + 'without cookies (200 and access control headers)', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + Cookie: 'testcookie=1', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); describe('CORS exposes headers', () => { @@ -809,17 +817,9 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], - ExposeHeaders: [ - 'x-amz-server-side-encryption', - 'x-amz-request-id', - 'x-amz-id-2', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], + ExposeHeaders: ['x-amz-server-side-encryption', 'x-amz-request-id', 'x-amz-id-2'], }, ], }, @@ -836,23 +836,23 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('if OPTIONS request matches CORS rule with ExposeHeader\'s, ' + - 'response should include Access-Control-Expose-Headers header', - done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-expose-headers': - 'x-amz-server-side-encryption, x-amz-request-id, x-amz-id-2', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + "if OPTIONS request matches CORS rule with ExposeHeader's, " + + 'response should include Access-Control-Expose-Headers header', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-expose-headers': 'x-amz-server-side-encryption, x-amz-request-id, x-amz-id-2', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); describe('CORS max age seconds', () => { @@ -861,12 +861,8 @@ describe('Preflight CORS request with existing bucket', () => { CORSConfiguration: { CORSRules: [ { - AllowedMethods: [ - 'GET', - ], - AllowedOrigins: [ - '*', - ], + AllowedMethods: ['GET'], + AllowedOrigins: ['*'], MaxAgeSeconds: 86400, }, ], @@ -884,20 +880,22 @@ describe('Preflight CORS request with existing bucket', () => { .catch(done); }); - it('if OPTIONS request matches CORS rule with max age seconds, ' + - 'response should include Access-Control-Max-Age header', done => { - const headers = { - 'Origin': allowedOrigin, - 'Access-Control-Request-Method': 'GET', - }; - const headersResponse = { - 'access-control-allow-origin': '*', - 'access-control-allow-methods': 'GET', - 'access-control-max-age': '86400', - vary, - }; - methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, - headersResponse }, done); - }); + it( + 'if OPTIONS request matches CORS rule with max age seconds, ' + + 'response should include Access-Control-Max-Age header', + done => { + const headers = { + Origin: allowedOrigin, + 'Access-Control-Request-Method': 'GET', + }; + const headersResponse = { + 'access-control-allow-origin': '*', + 'access-control-allow-methods': 'GET', + 'access-control-max-age': '86400', + vary, + }; + methodRequest({ method: 'OPTIONS', bucket, headers, code: 200, headersResponse }, done); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/deleteMpu.js b/tests/functional/aws-node-sdk/test/object/deleteMpu.js index 60fdad6f13..d6a054046a 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteMpu.js +++ b/tests/functional/aws-node-sdk/test/object/deleteMpu.js @@ -18,8 +18,8 @@ const westLocation = 'scality-us-west-1'; const eastLocation = 'us-east-1'; const confLocations = [ - { name: 'us-west-1', statusCode: 204, location: westLocation, describe }, - { name: 'us-east-1', statusCode: 404, location: eastLocation, describe }, + { name: 'us-west-1', statusCode: 204, location: westLocation, describe }, + { name: 'us-east-1', statusCode: 404, location: eastLocation, describe }, ]; describe('DELETE multipart', () => { @@ -34,19 +34,17 @@ describe('DELETE multipart', () => { UploadId: uploadId, }); - s3Client.send(command) + s3Client + .send(command) .then(response => { - const statusCode = - response?.$metadata?.httpStatusCode; - assert.strictEqual(statusCode, statusCodeExpected, - `Found unexpected statusCode ${statusCode}`); + const statusCode = response?.$metadata?.httpStatusCode; + assert.strictEqual(statusCode, statusCodeExpected, `Found unexpected statusCode ${statusCode}`); return callback(); }) .catch(err => { const statusCode = err?.$metadata?.httpStatusCode; if (statusCode) { - assert.strictEqual(statusCode, statusCodeExpected, - `Found unexpected statusCode ${statusCode}`); + assert.strictEqual(statusCode, statusCodeExpected, `Found unexpected statusCode ${statusCode}`); } if (statusCodeExpected === 204) { return callback(err); @@ -55,8 +53,7 @@ describe('DELETE multipart', () => { }); } - it('on bucket that does not exist: should return NoSuchBucket', - done => { + it('on bucket that does not exist: should return NoSuchBucket', done => { const uploadId = 'nonexistinguploadid'; const command = new AbortMultipartUploadCommand({ Bucket: bucket, @@ -64,22 +61,20 @@ describe('DELETE multipart', () => { UploadId: uploadId, }); - s3Client.send(command) + s3Client + .send(command) .then(() => { done(new Error('Expected NoSuchBucket but request succeeded')); }) .catch(err => { - assert.notEqual(err, null, - 'Expected NoSuchBucket but found no err'); + assert.notEqual(err, null, 'Expected NoSuchBucket but found no err'); assert.strictEqual(err.name, 'NoSuchBucket'); done(); }); }); confLocations.forEach(confLocation => { - confLocation.describe('on existing bucket with ' + - `${confLocation.name}`, - () => { + confLocation.describe('on existing bucket with ' + `${confLocation.name}`, () => { beforeEach(async () => { const command = new CreateBucketCommand({ Bucket: bucket, @@ -91,38 +86,38 @@ describe('DELETE multipart', () => { }); afterEach(async () => { - process.stdout.write('Emptying bucket\n'); - await bucketUtil.empty(bucket); - process.stdout.write('Deleting bucket\n'); - await bucketUtil.deleteOne(bucket); + process.stdout.write('Emptying bucket\n'); + await bucketUtil.empty(bucket); + process.stdout.write('Deleting bucket\n'); + await bucketUtil.deleteOne(bucket); }); - itSkipIfAWS(`should return ${confLocation.statusCode} if ` + - 'mpu does not exist with uploadId', - done => { - const uploadId = 'nonexistinguploadid'; - _assertStatusCode(uploadId, confLocation.statusCode, done); - }); + itSkipIfAWS( + `should return ${confLocation.statusCode} if ` + 'mpu does not exist with uploadId', + done => { + const uploadId = 'nonexistinguploadid'; + _assertStatusCode(uploadId, confLocation.statusCode, done); + }, + ); - describe('if mpu exists with uploadId + at least one part', - () => { + describe('if mpu exists with uploadId + at least one part', () => { let uploadId; beforeEach(async () => { - const createCommand = new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - }); - const createResponse = await s3Client.send(createCommand); - uploadId = createResponse.UploadId; - const uploadCommand = new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: 1, - UploadId: uploadId, - Body: Buffer.from('test data'), - }); - await s3Client.send(uploadCommand); + const createCommand = new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }); + const createResponse = await s3Client.send(createCommand); + uploadId = createResponse.UploadId; + const uploadCommand = new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: Buffer.from('test data'), + }); + await s3Client.send(uploadCommand); }); it('should return 204 for abortMultipartUpload', done => { diff --git a/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js b/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js index d8839a5760..da40f392a1 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/deleteObjTagging.js @@ -16,16 +16,18 @@ const bucketName = 'testdeletetaggingbucket'; const objectName = 'testtaggingobject'; const objectNameAcl = 'testtaggingobjectacl'; -const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }, - { - Key: 'key2', - Value: 'value2', - }, -] }; +const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + { + Key: 'key2', + Value: 'value2', + }, + ], +}; function _checkError(err, code, statusCode) { assert(err, 'Expected error but found none'); @@ -53,120 +55,155 @@ describe('DELETE object taggings', () => { }); it('should delete tag set', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); - const dataGet = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const dataGet = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.strictEqual(dataGet.TagSet.length, 0); }); it('should delete a non-existing tag set', async () => { - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); - const dataGet = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const dataGet = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.strictEqual(dataGet.TagSet.length, 0); }); - it('should return NoSuchKey deleting tag set to a non-existing object', - async () => { + it('should return NoSuchKey deleting tag set to a non-existing object', async () => { try { - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: 'nonexisting', - })); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: 'nonexisting', + }), + ); assert.fail('Expected NoSuchKey error'); } catch (err) { _checkError(err, 'NoSuchKey', 404); } }); - it('should return 403 AccessDenied deleting tag set with another ' + - 'account', async () => { + it('should return 403 AccessDenied deleting tag set with another ' + 'account', async () => { try { - await otherAccountS3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); + await otherAccountS3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.fail('Expected AccessDenied error'); } catch (err) { _checkError(err, 'AccessDenied', 403); } }); - it('should return 403 AccessDenied deleting tag set with a different ' + - 'account to an object with ACL "public-read-write"', + it( + 'should return 403 AccessDenied deleting tag set with a different ' + + 'account to an object with ACL "public-read-write"', async () => { - await s3.send(new PutObjectAclCommand({ - Bucket: bucketName, - Key: objectName, - ACL: 'public-read-write' - })); - - try { - await otherAccountS3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName - })); - assert.fail('Expected AccessDenied error'); - } catch (err) { - _checkError(err, 'AccessDenied', 403); - } - }); + await s3.send( + new PutObjectAclCommand({ + Bucket: bucketName, + Key: objectName, + ACL: 'public-read-write', + }), + ); + + try { + await otherAccountS3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + assert.fail('Expected AccessDenied error'); + } catch (err) { + _checkError(err, 'AccessDenied', 403); + } + }, + ); - it('should return 403 AccessDenied deleting tag set to an object '+ - ' in a bucket created with a different account', + it( + 'should return 403 AccessDenied deleting tag set to an object ' + + ' in a bucket created with a different account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write' - })); - - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - - try { - await otherAccountS3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - assert.fail('Expected AccessDenied error'); - } catch (err) { - _checkError(err, 'AccessDenied', 403); - } - }); + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); - it('should delete tag set to an object in a bucket created with '+ - 'same account even though object put by other account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write' - })); - - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl - })); - }); + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + + try { + await otherAccountS3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + assert.fail('Expected AccessDenied error'); + } catch (err) { + _checkError(err, 'AccessDenied', 403); + } + }, + ); + + it( + 'should delete tag set to an object in a bucket created with ' + + 'same account even though object put by other account', + async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/deleteObject.js b/tests/functional/aws-node-sdk/test/object/deleteObject.js index ee14d27d06..ccfd453af5 100644 --- a/tests/functional/aws-node-sdk/test/object/deleteObject.js +++ b/tests/functional/aws-node-sdk/test/object/deleteObject.js @@ -1,7 +1,7 @@ const assert = require('assert'); const moment = require('moment'); const { - CreateBucketCommand, + CreateBucketCommand, CreateMultipartUploadCommand, UploadPartCommand, CompleteMultipartUploadCommand, @@ -10,7 +10,7 @@ const { PutObjectRetentionCommand, PutObjectLegalHoldCommand, PutObjectLockConfigurationCommand, - HeadObjectCommand + HeadObjectCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -18,7 +18,6 @@ const changeObjectLock = require('../../../../utilities/objectLock-util'); const objectName = 'key'; const objectNameTwo = 'secondkey'; - describe('DELETE object', () => { withV4(sigCfg => { let uploadId; @@ -32,13 +31,15 @@ describe('DELETE object', () => { try { process.stdout.write('creating bucket\n'); await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - + process.stdout.write('initiating multipart upload\n'); - const createRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: objectName, - })); - + const createRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + process.stdout.write('uploading parts\n'); uploadId = createRes.UploadId; // Concurrent uploads help trigger flakiness with "TimeoutError: @@ -47,37 +48,43 @@ describe('DELETE object', () => { const uploadWithRetry = (params, attempt = 0) => s3.send(new UploadPartCommand(params)).catch(err => { if (attempt < 3) { - process.stdout.write(`Retrying UploadPart ${params.PartNumber} ` - + `(attempt ${attempt + 1}/3): ${err}\n`); + process.stdout.write( + `Retrying UploadPart ${params.PartNumber} ` + + `(attempt ${attempt + 1}/3): ${err}\n`, + ); return uploadWithRetry(params, attempt + 1); } throw err; }); const uploads = []; for (let i = 1; i <= 3; i++) { - uploads.push(uploadWithRetry({ - Bucket: bucketName, - Key: objectName, - PartNumber: i, - Body: testfile, - UploadId: uploadId, - })); + uploads.push( + uploadWithRetry({ + Bucket: bucketName, + Key: objectName, + PartNumber: i, + Body: testfile, + UploadId: uploadId, + }), + ); } const uploadResults = await Promise.all(uploads); - + process.stdout.write('about to complete multipart upload\n'); - await s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: objectName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: uploadResults[0].ETag, PartNumber: 1 }, - { ETag: uploadResults[1].ETag, PartNumber: 2 }, - { ETag: uploadResults[2].ETag, PartNumber: 3 }, - ], - }, - })); + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { ETag: uploadResults[0].ETag, PartNumber: 1 }, + { ETag: uploadResults[1].ETag, PartNumber: 2 }, + { ETag: uploadResults[2].ETag, PartNumber: 3 }, + ], + }, + }), + ); } catch (err) { process.stdout.write(`Error in before: ${err}\n`); throw err; @@ -112,47 +119,57 @@ describe('DELETE object', () => { let versionIdOne; let versionIdTwo; const retainDate = moment().add(10, 'days').toDate(); - + before(async () => { try { process.stdout.write('creating bucket\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); - + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); + process.stdout.write('putting object\n'); - const res1 = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })); + const res1 = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); versionIdOne = res1.VersionId; - + process.stdout.write('putting object retention\n'); - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: { - Mode: 'GOVERNANCE', - RetainUntilDate: retainDate, - }, - })); - + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: { + Mode: 'GOVERNANCE', + RetainUntilDate: retainDate, + }, + }), + ); + process.stdout.write('putting object\n'); - const res2 = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameTwo, - })); + const res2 = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameTwo, + }), + ); versionIdTwo = res2.VersionId; - + process.stdout.write('putting object legal hold\n'); - await s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucketName, - Key: objectNameTwo, - LegalHold: { - Status: 'ON', - }, - })); + await s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucketName, + Key: objectNameTwo, + LegalHold: { + Status: 'ON', + }, + }), + ); } catch (err) { process.stdout.write(`Error in before: ${err}\n`); throw err; @@ -165,10 +182,12 @@ describe('DELETE object', () => { }); it('should put delete marker if no version id specified', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - })) + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) .then(() => { done(); }) @@ -179,11 +198,13 @@ describe('DELETE object', () => { }); it('should not delete object version locked with object retention', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionIdOne, - })) + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionIdOne, + }), + ) .then(() => { assert.fail('Should have failed'); }) @@ -194,12 +215,14 @@ describe('DELETE object', () => { }); it('should delete locked object version with GOVERNANCE retention mode and correct header', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionIdOne, - BypassGovernanceRetention: true, - })) + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionIdOne, + BypassGovernanceRetention: true, + }), + ) .then(() => { done(); }) @@ -210,19 +233,25 @@ describe('DELETE object', () => { }); it('should not delete object locked with legal hold', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectNameTwo, - VersionId: versionIdTwo, - })) - .catch(err => { + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectNameTwo, + VersionId: versionIdTwo, + }), + ).catch(err => { assert.strictEqual(err.name, 'AccessDenied'); changeObjectLock( - [{ - bucket: bucketName, - key: objectNameTwo, - versionId: versionIdTwo, - }], '', done); + [ + { + bucket: bucketName, + key: objectNameTwo, + versionId: versionIdTwo, + }, + ], + '', + done, + ); }); }); }); @@ -231,44 +260,52 @@ describe('DELETE object', () => { const bucketName = 'testdeletelocklegalholdbucket'; const objectName = 'key'; let versionId; - + before(async () => { try { process.stdout.write('creating bucket\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); - + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); + process.stdout.write('putting object lock configuration\n'); - await s3.send(new PutObjectLockConfigurationCommand({ - Bucket: bucketName, - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { - DefaultRetention: { - Mode: 'GOVERNANCE', - Days: 1, + await s3.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucketName, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Mode: 'GOVERNANCE', + Days: 1, + }, }, }, - }, - })); - + }), + ); + process.stdout.write('putting object\n'); - const res = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); versionId = res.VersionId; - + process.stdout.write('putting object legal hold\n'); - await s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucketName, - Key: objectName, - LegalHold: { - Status: 'ON', - }, - })); + await s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucketName, + Key: objectName, + LegalHold: { + Status: 'ON', + }, + }), + ); } catch (err) { process.stdout.write(`Error in before: ${err}\n`); throw err; @@ -287,24 +324,33 @@ describe('DELETE object', () => { } }); - it('should not delete locked object version with GOVERNANCE ' + - 'retention mode and bypass header when object is legal-hold enabled', done => { - s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - BypassGovernanceRetention: true, - })) - .catch(err => { - assert.strictEqual(err.name, 'AccessDenied'); - changeObjectLock( - [{ - bucket: bucketName, - key: objectName, - versionId, - }], '', done); - }); - }); + it( + 'should not delete locked object version with GOVERNANCE ' + + 'retention mode and bypass header when object is legal-hold enabled', + done => { + s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + BypassGovernanceRetention: true, + }), + ).catch(err => { + assert.strictEqual(err.name, 'AccessDenied'); + changeObjectLock( + [ + { + bucket: bucketName, + key: objectName, + versionId, + }, + ], + '', + done, + ); + }); + }, + ); }); describe('with conditional headers (unofficial, for backbeat)', () => { @@ -319,15 +365,19 @@ describe('DELETE object', () => { beforeEach(async () => { // Re-create the object for each test since some tests will delete it - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: testObjectKey, - Body: testObjectBody, - })); - const head = await s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: testObjectKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + Body: testObjectBody, + }), + ); + const head = await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ); objectLastModified = head.LastModified; }); @@ -340,7 +390,7 @@ describe('DELETE object', () => { const command = new DeleteObjectCommand(params); // Create a unique middleware name to avoid conflicts const middlewareName = `headersAdder_${Date.now()}_${Math.random()}`; - + // Middleware to add custom headers const middleware = next => async args => { for (const [key, value] of Object.entries(headers)) { @@ -350,15 +400,15 @@ describe('DELETE object', () => { } return next(args); }; - + const middlewareConfig = { step: 'build', name: middlewareName, }; - + // Add middleware s3.middlewareStack.add(middleware, middlewareConfig); - + s3.send(command) .then(data => { s3.middlewareStack.remove(middlewareName); @@ -374,41 +424,53 @@ describe('DELETE object', () => { it('should delete when condition is true (date after object modification)', done => { const futureDate = new Date(objectLastModified.getTime() + 60_000); // 1 minute later - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Unmodified-Since': futureDate.toUTCString(), - }, (err, data) => { - assert.ifError(err); - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - s3.send(new HeadObjectCommand({ + deleteObjectConditional( + s3, + { Bucket: bucketName, Key: testObjectKey, - })) - .then(() => { - assert.fail('Object should not exist'); - }) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - done(); - }); - }); + }, + { + 'If-Unmodified-Since': futureDate.toUTCString(), + }, + (err, data) => { + assert.ifError(err); + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ) + .then(() => { + assert.fail('Object should not exist'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotFound'); + done(); + }); + }, + ); }); it('should fail (412) to delete when condition is false (date before object modification)', done => { const pastDate = new Date(objectLastModified.getTime() - 60_000); // 1 minute earlier - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Unmodified-Since': pastDate.toUTCString(), - }, err => { - assert.strictEqual(err.name, 'PreconditionFailed'); - assert.strictEqual(err.$metadata.httpStatusCode, 412); - done(); - }); + deleteObjectConditional( + s3, + { + Bucket: bucketName, + Key: testObjectKey, + }, + { + 'If-Unmodified-Since': pastDate.toUTCString(), + }, + err => { + assert.strictEqual(err.name, 'PreconditionFailed'); + assert.strictEqual(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); }); @@ -416,36 +478,47 @@ describe('DELETE object', () => { it('should delete when condition is true (date before object modification)', done => { const pastDate = new Date(objectLastModified.getTime() - 60_000); // 1 minute earlier - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Modified-Since': pastDate.toUTCString() - }, (err, data) => { - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - s3.send(new HeadObjectCommand({ + deleteObjectConditional( + s3, + { Bucket: bucketName, Key: testObjectKey, - })) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - done(); - }); - }); + }, + { + 'If-Modified-Since': pastDate.toUTCString(), + }, + (err, data) => { + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ).catch(err => { + assert.strictEqual(err.name, 'NotFound'); + done(); + }); + }, + ); }); it('should fail (304) to delete when condition is false (date after object modification)', done => { const futureDate = new Date(objectLastModified.getTime() + 60_000); // 1 minute later - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Modified-Since': futureDate.toUTCString(), - }, err => { - assert.strictEqual(err.$metadata.httpStatusCode, 304); - done(); - }); + deleteObjectConditional( + s3, + { + Bucket: bucketName, + Key: testObjectKey, + }, + { + 'If-Modified-Since': futureDate.toUTCString(), + }, + err => { + assert.strictEqual(err.$metadata.httpStatusCode, 304); + done(); + }, + ); }); }); @@ -454,24 +527,30 @@ describe('DELETE object', () => { const pastDate = new Date(objectLastModified.getTime() - 60_000); // 1 minute earlier const futureDate = new Date(objectLastModified.getTime() + 60_000); // 1 minute later - deleteObjectConditional(s3, { - Bucket: bucketName, - Key: testObjectKey, - }, { - 'If-Modified-Since': pastDate.toUTCString(), - 'If-Unmodified-Since': futureDate.toUTCString(), - }, (err, data) => { - assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); - s3.send(new HeadObjectCommand({ + deleteObjectConditional( + s3, + { Bucket: bucketName, Key: testObjectKey, - })) - .catch(err => { - assert.strictEqual(err.name, 'NotFound'); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - done(); - }); - }); + }, + { + 'If-Modified-Since': pastDate.toUTCString(), + 'If-Unmodified-Since': futureDate.toUTCString(), + }, + (err, data) => { + assert.deepStrictEqual(data.$metadata.httpStatusCode, 204); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: testObjectKey, + }), + ).catch(err => { + assert.strictEqual(err.name, 'NotFound'); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + done(); + }); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js b/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js index d4ec55375a..3a765a224e 100644 --- a/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/encryptionHeaders.js @@ -1,7 +1,7 @@ const assert = require('assert'); const async = require('async'); const uuid = require('uuid'); -const { +const { CreateBucketCommand, HeadObjectCommand, PutObjectCommand, @@ -37,16 +37,20 @@ const testCases = [ }, ]; -function s3NoOp(_, cb) { cb(); } +function s3NoOp(_, cb) { + cb(); +} function getSSEConfig(s3, Bucket, Key, cb) { const command = new HeadObjectCommand({ Bucket, Key }); s3.send(command) .then(resp => { - const sseConfig = JSON.parse(JSON.stringify({ - algo: resp.ServerSideEncryption, - masterKeyId: resp.SSEKMSKeyId - })); + const sseConfig = JSON.parse( + JSON.stringify({ + algo: resp.ServerSideEncryption, + masterKeyId: resp.SSEKMSKeyId, + }), + ); cb(null, sseConfig); }) .catch(cb); @@ -58,15 +62,15 @@ function putEncryptedObject(s3, Bucket, Key, sseConfig, kmsKeyId, cb) { Key, Body: 'somedata', }; - + if (sseConfig.algo) { params.ServerSideEncryption = sseConfig.algo; } - + if (sseConfig.masterKeyId) { params.SSEKMSKeyId = kmsKeyId; } - + const command = new PutObjectCommand(params); s3.send(command) .then(response => cb(null, response)) @@ -80,9 +84,7 @@ function createExpected(sseConfig, kmsKeyId) { } if (sseConfig.masterKeyId) { - expected.masterKeyId = config.kmsHideScalityArn - ? getKeyIdFromArn(kmsKeyId) - : kmsKeyId; + expected.masterKeyId = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKeyId) : kmsKeyId; } return expected; } @@ -99,8 +101,7 @@ function hydrateSSEConfig({ algo: SSEAlgorithm, masterKeyId: KMSMasterKeyID }) { }, }, ], - } - ) + }), ); } @@ -143,15 +144,12 @@ describe('per object encryption headers', () => { let kmsKeyId; before(done => { - const bucket = new BucketInfo('enc-bucket-test', 'OwnerId', - 'OwnerDisplayName', new Date().toJSON()); - kms.createBucketKey(bucket, log, - (err, { masterKeyArn: keyId }) => { - assert.ifError(err); - kmsKeyId = keyId; - done(); - } - ); + const bucket = new BucketInfo('enc-bucket-test', 'OwnerId', 'OwnerDisplayName', new Date().toJSON()); + kms.createBucketKey(bucket, log, (err, { masterKeyArn: keyId }) => { + assert.ifError(err); + kmsKeyId = keyId; + done(); + }); }); beforeEach(async () => { @@ -177,30 +175,30 @@ describe('per object encryption headers', () => { putEncryptedObject(s3, bucket, object, target, kmsKeyId, (error, putResp) => { assert.ifError(error); if (target.algo) { - assert.strictEqual(putResp.ServerSideEncryption, target.algo, - 'PutObject response should include ServerSideEncryption header'); + assert.strictEqual( + putResp.ServerSideEncryption, + target.algo, + 'PutObject response should include ServerSideEncryption header', + ); if (target.algo === 'aws:kms') { - assert(putResp.SSEKMSKeyId, - 'PutObject response should include SSEKMSKeyId for aws:kms'); + assert( + putResp.SSEKMSKeyId, + 'PutObject response should include SSEKMSKeyId for aws:kms', + ); } } - return getSSEConfig( - s3, - bucket, - object, - (error, sseConfig) => { - assert.ifError(error); - const expected = createExpected(target, kmsKeyId); - // We differ from aws behavior and always return a - // masterKeyId even when not explicitly configured. - if (expected.algo === 'aws:kms' && !expected.masterKeyId) { - // eslint-disable-next-line no-param-reassign - delete sseConfig.masterKeyId; - } - assert.deepStrictEqual(sseConfig, expected); - done(); + return getSSEConfig(s3, bucket, object, (error, sseConfig) => { + assert.ifError(error); + const expected = createExpected(target, kmsKeyId); + // We differ from aws behavior and always return a + // masterKeyId even when not explicitly configured. + if (expected.algo === 'aws:kms' && !expected.masterKeyId) { + // eslint-disable-next-line no-param-reassign + delete sseConfig.masterKeyId; } - ); + assert.deepStrictEqual(sseConfig, expected); + done(); + }); })); it('should put two encrypted objects in a unencrypted bucket, reusing the generated config', done => @@ -223,48 +221,52 @@ describe('per object encryption headers', () => { } res.forEach(sseConfig => assert.deepStrictEqual(sseConfig, expected)); done(); - } + }, ); - } + }, )); - testCases - .forEach(existing => { + testCases.forEach(existing => { const hasKey = target.masterKeyId ? 'a' : 'no'; const { algo } = target; - it('should override bucket encryption settings with ' - + `algo ${algo || 'none'} with ${hasKey} key id`, done => { - const _existing = Object.assign({}, existing); - if (existing.masterKeyId) { - _existing.masterKeyId = kmsKeyId; - } - const params = { - Bucket: bucket, - ServerSideEncryptionConfiguration: hydrateSSEConfig(_existing), - }; - // no op putBucketEncryption for the unencrypted case - const s3Op = existing.algo ? - (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; - s3Op(params, error => { - assert.ifError(error); - return putEncryptedObject(s3, bucket, object, target, kmsKeyId, (error, putResp) => { + it( + 'should override bucket encryption settings with ' + + `algo ${algo || 'none'} with ${hasKey} key id`, + done => { + const _existing = Object.assign({}, existing); + if (existing.masterKeyId) { + _existing.masterKeyId = kmsKeyId; + } + const params = { + Bucket: bucket, + ServerSideEncryptionConfiguration: hydrateSSEConfig(_existing), + }; + // no op putBucketEncryption for the unencrypted case + const s3Op = existing.algo ? (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; + s3Op(params, error => { assert.ifError(error); - if (target.algo) { - assert.strictEqual(putResp.ServerSideEncryption, target.algo, - 'PutObject response should include ServerSideEncryption header'); - if (target.algo === 'aws:kms') { - assert(putResp.SSEKMSKeyId, - 'PutObject response should include SSEKMSKeyId for aws:kms'); + return putEncryptedObject(s3, bucket, object, target, kmsKeyId, (error, putResp) => { + assert.ifError(error); + if (target.algo) { + assert.strictEqual( + putResp.ServerSideEncryption, + target.algo, + 'PutObject response should include ServerSideEncryption header', + ); + if (target.algo === 'aws:kms') { + assert( + putResp.SSEKMSKeyId, + 'PutObject response should include SSEKMSKeyId for aws:kms', + ); + } + } else if (existing.algo) { + assert.strictEqual( + putResp.ServerSideEncryption, + existing.algo, + 'PutObject response should include ServerSideEncryption from bucket default', + ); } - } else if (existing.algo) { - assert.strictEqual(putResp.ServerSideEncryption, existing.algo, - 'PutObject response should include ServerSideEncryption from bucket default'); - } - return getSSEConfig( - s3, - bucket, - object, - (error, sseConfig) => { + return getSSEConfig(s3, bucket, object, (error, sseConfig) => { assert.ifError(error); let expected = createExpected(target, kmsKeyId); // In the null case the expected encryption config is @@ -280,29 +282,25 @@ describe('per object encryption headers', () => { } assert.deepStrictEqual(sseConfig, expected); done(); - } - ); + }); + }); }); - }); - }); + }, + ); }); - testCases - .forEach(existing => it('should copy an object to an encrypted key overriding bucket settings', - done => { + testCases.forEach(existing => + it('should copy an object to an encrypted key overriding bucket settings', done => { const _existing = Object.assign({}, existing); if (existing.masterKeyId) { - _existing.masterKeyId = config.kmsHideScalityArn - ? getKeyIdFromArn(kmsKeyId) - : kmsKeyId; + _existing.masterKeyId = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKeyId) : kmsKeyId; } const params = { Bucket: bucket2, ServerSideEncryptionConfiguration: hydrateSSEConfig(_existing), }; // no op putBucketEncryption for the unencrypted case - const s3Op = existing.algo ? - (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; + const s3Op = existing.algo ? (params, cb) => putBucketEncryption(s3, params, cb) : s3NoOp; s3Op(params, error => { assert.ifError(error); return putEncryptedObject(s3, bucket, object, target, kmsKeyId, error => { @@ -320,32 +318,28 @@ describe('per object encryption headers', () => { } return copyObject(s3, copyParams, error => { assert.ifError(error); - return getSSEConfig( - s3, - bucket2, - object2, - (error, sseConfig) => { - assert.ifError(error); - let expected = createExpected(target, kmsKeyId); - // In the null case the expected encryption config is - // the buckets default policy - if (!target.algo) { - expected = _existing; - } - // We differ from aws behavior and always return a - // masterKeyId even when not explicitly configured. - if (expected.algo === 'aws:kms' && !expected.masterKeyId) { + return getSSEConfig(s3, bucket2, object2, (error, sseConfig) => { + assert.ifError(error); + let expected = createExpected(target, kmsKeyId); + // In the null case the expected encryption config is + // the buckets default policy + if (!target.algo) { + expected = _existing; + } + // We differ from aws behavior and always return a + // masterKeyId even when not explicitly configured. + if (expected.algo === 'aws:kms' && !expected.masterKeyId) { // eslint-disable-next-line no-param-reassign - delete sseConfig.masterKeyId; - } - assert.deepStrictEqual(sseConfig, expected); - done(); + delete sseConfig.masterKeyId; } - ); + assert.deepStrictEqual(sseConfig, expected); + done(); + }); }); }); }); - })); + }), + ); it('should init an encrypted MPU and put an encrypted part', done => { const params = { diff --git a/tests/functional/aws-node-sdk/test/object/get.js b/tests/functional/aws-node-sdk/test/object/get.js index 3acabdfa19..7c4527b2c8 100644 --- a/tests/functional/aws-node-sdk/test/object/get.js +++ b/tests/functional/aws-node-sdk/test/object/get.js @@ -49,8 +49,7 @@ const etag = `"${etagTrim}"`; const partSize = 1024 * 1024 * 5; // 5MB minumum required part size. function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function checkError(err, code) { @@ -78,145 +77,202 @@ describe('GET object', () => { let s3; function requestGet(fields, cb) { - s3.send(new GetObjectCommand(Object.assign({ - Bucket: bucketName, - Key: objectName, - }, fields))).then(data => cb(null, data)).catch(err => { - if (err.$metadata.httpStatusCode === 304) { - const notModifiedError = new Error('NotModified'); - notModifiedError.name = 'NotModified'; - notModifiedError.$metadata = err.$metadata; - return cb(notModifiedError); - } - return cb(err); - }); + s3.send( + new GetObjectCommand( + Object.assign( + { + Bucket: bucketName, + Key: objectName, + }, + fields, + ), + ), + ) + .then(data => cb(null, data)) + .catch(err => { + if (err.$metadata.httpStatusCode === 304) { + const notModifiedError = new Error('NotModified'); + notModifiedError.name = 'NotModified'; + notModifiedError.$metadata = err.$metadata; + return cb(notModifiedError); + } + return cb(err); + }); } const requestGetPromise = promisify(requestGet); function checkGetObjectPart(key, partNumber, len, body, cb) { - s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: key, - PartNumber: partNumber, - })).then(async data => { - checkIntegerHeader(data.ContentLength, len); - const md5Hash = crypto.createHash('md5'); - const md5HashExpected = crypto.createHash('md5'); - const bodyText = await data.Body.transformToString(); - assert.strictEqual( - md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(body).digest('hex') - ); - return cb(); - }).catch(cb); + s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: key, + PartNumber: partNumber, + }), + ) + .then(async data => { + checkIntegerHeader(data.ContentLength, len); + const md5Hash = crypto.createHash('md5'); + const md5HashExpected = crypto.createHash('md5'); + const bodyText = await data.Body.transformToString(); + assert.strictEqual( + md5Hash.update(bodyText).digest('hex'), + md5HashExpected.update(body).digest('hex'), + ); + return cb(); + }) + .catch(cb); } // Upload parts with the given partNumbers array and complete MPU. function completeMPU(partNumbers, cb) { let ETags = []; - return async.waterfall([ - next => { - const createMpuParams = { - Bucket: bucketName, - Key: objectName, - }; - - s3.send(new CreateMultipartUploadCommand(createMpuParams)).then(data => - next(null, data.UploadId)).catch(next); - }, - (uploadId, next) => - async.eachSeries(partNumbers, (partNumber, callback) => { - const uploadPartParams = { + return async.waterfall( + [ + next => { + const createMpuParams = { Bucket: bucketName, Key: objectName, - PartNumber: partNumber, - UploadId: uploadId, - Body: Buffer.alloc(partSize).fill(partNumber), }; - return s3.send(new UploadPartCommand(uploadPartParams)).then(data => { - ETags = ETags.concat(data.ETag); - return callback(); - }).catch(callback); - }, err => next(err, uploadId)), - (uploadId, next) => { - const parts = Array.from(Array(partNumbers.length).keys()); - const params = { - Bucket: bucketName, - Key: objectName, - MultipartUpload: { - Parts: parts.map(n => ({ - ETag: ETags[n], - PartNumber: partNumbers[n], - })), - }, - UploadId: uploadId, - }; - return s3.send(new CompleteMultipartUploadCommand(params)).then(() => - next(null, uploadId)).catch(next); - }, - ], (err, uploadId) => { - if (err) { - if (uploadId) { - return s3.send(new AbortMultipartUploadCommand({ + + s3.send(new CreateMultipartUploadCommand(createMpuParams)) + .then(data => next(null, data.UploadId)) + .catch(next); + }, + (uploadId, next) => + async.eachSeries( + partNumbers, + (partNumber, callback) => { + const uploadPartParams = { + Bucket: bucketName, + Key: objectName, + PartNumber: partNumber, + UploadId: uploadId, + Body: Buffer.alloc(partSize).fill(partNumber), + }; + return s3 + .send(new UploadPartCommand(uploadPartParams)) + .then(data => { + ETags = ETags.concat(data.ETag); + return callback(); + }) + .catch(callback); + }, + err => next(err, uploadId), + ), + (uploadId, next) => { + const parts = Array.from(Array(partNumbers.length).keys()); + const params = { Bucket: bucketName, Key: objectName, + MultipartUpload: { + Parts: parts.map(n => ({ + ETag: ETags[n], + PartNumber: partNumbers[n], + })), + }, UploadId: uploadId, - })).then(() => cb(err)).catch(() => cb(err)); + }; + return s3 + .send(new CompleteMultipartUploadCommand(params)) + .then(() => next(null, uploadId)) + .catch(next); + }, + ], + (err, uploadId) => { + if (err) { + if (uploadId) { + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + }), + ) + .then(() => cb(err)) + .catch(() => cb(err)); + } + return cb(err); } - return cb(err); - } - return cb(); - }); + return cb(); + }, + ); } function createMPUAndPutTwoParts(partTwoBody, cb) { let uploadId; const ETags = []; - return async.waterfall([ - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - })).then(data => { - uploadId = data.UploadId; - return next(); - }).catch(next), - // Copy an object with three parts. - next => s3.send(new UploadPartCopyCommand({ - Bucket: bucketName, - CopySource: `/${bucketName}/${objectName}`, - Key: copyPartKey, - PartNumber: 1, - UploadId: uploadId, - })).then(data => { - ETags[0] = data.CopyPartResult.ETag; - return next(); - }).catch(next), - // Put an object with one part. - next => s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: copyPartKey, - PartNumber: 2, - UploadId: uploadId, - Body: partTwoBody, - })).then(data => { - ETags[1] = data.ETag; - return next(); - }).catch(next), - ], err => { - if (err) { - if (uploadId) { - return s3.send(new AbortMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - UploadId: uploadId, - })).then(() => cb(err)).catch(() => cb(err)); + return async.waterfall( + [ + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + }), + ) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + // Copy an object with three parts. + next => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: bucketName, + CopySource: `/${bucketName}/${objectName}`, + Key: copyPartKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + ETags[0] = data.CopyPartResult.ETag; + return next(); + }) + .catch(next), + // Put an object with one part. + next => + s3 + .send( + new UploadPartCommand({ + Bucket: bucketName, + Key: copyPartKey, + PartNumber: 2, + UploadId: uploadId, + Body: partTwoBody, + }), + ) + .then(data => { + ETags[1] = data.ETag; + return next(); + }) + .catch(next), + ], + err => { + if (err) { + if (uploadId) { + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + UploadId: uploadId, + }), + ) + .then(() => cb(err)) + .catch(() => cb(err)); + } + return cb(err); } - return cb(err); - } - return cb(null, uploadId, ETags); - }); + return cb(null, uploadId, ETags); + }, + ); } before(async () => { @@ -227,98 +283,96 @@ describe('GET object', () => { }); after(async () => { - await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); - await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); + await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return NoSuchKey error when no such object', - done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'nope' })).then(() => { + it('should return NoSuchKey error when no such object', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'nope' })) + .then(() => { assert.fail('Expected failure but got success'); - }).catch(err => { + }) + .catch(err => { assert.strictEqual(err.name, 'NoSuchKey'); return done(); }); - }); + }); - it('should return NoSuchKey error when no such object even with key longer than 915 bytes', - done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'a'.repeat(2000) })).then(() => { + it('should return NoSuchKey error when no such object even with key longer than 915 bytes', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: 'a'.repeat(2000) })) + .then(() => { assert.fail('Expected failure but got success'); - }).catch(err => { + }) + .catch(err => { assert.strictEqual(err.name, 'NoSuchKey'); return done(); }); - }); + }); - describe('Additional headers: [Cache-Control, Content-Disposition, ' + - 'Content-Encoding, Expires, Accept-Ranges]', () => { - describe('if specified in put object request', () => { - before(async () => { - const params = { - Bucket: bucketName, - Key: objectName, - CacheControl: cacheControl, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - ContentType: contentType, - Expires: expires, - }; - await s3.send(new PutObjectCommand(params)); - }); - it('should return additional headers', done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })).then(res => { - assert.strictEqual(res.CacheControl, - cacheControl); - assert.strictEqual(res.ContentDisposition, - contentDisposition); - // Should remove V4 streaming value 'aws-chunked' - // to be compatible with AWS behavior - assert.strictEqual(res.ContentEncoding, - 'gzip'); - assert.strictEqual(res.ContentType, contentType); - assert.strictEqual(res.Expires.toGMTString(), - new Date(expires).toGMTString()); - assert.strictEqual(res.AcceptRanges, 'bytes'); - return done(); - }).catch(done); + describe( + 'Additional headers: [Cache-Control, Content-Disposition, ' + 'Content-Encoding, Expires, Accept-Ranges]', + () => { + describe('if specified in put object request', () => { + before(async () => { + const params = { + Bucket: bucketName, + Key: objectName, + CacheControl: cacheControl, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + ContentType: contentType, + Expires: expires, + }; + await s3.send(new PutObjectCommand(params)); + }); + it('should return additional headers', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + // Should remove V4 streaming value 'aws-chunked' + // to be compatible with AWS behavior + assert.strictEqual(res.ContentEncoding, 'gzip'); + assert.strictEqual(res.ContentType, contentType); + assert.strictEqual(res.Expires.toGMTString(), new Date(expires).toGMTString()); + assert.strictEqual(res.AcceptRanges, 'bytes'); + return done(); + }) + .catch(done); + }); }); - }); - describe('if response content headers are set in query', () => { - before(async () => { - await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); - }); + describe('if response content headers are set in query', () => { + before(async () => { + await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); + }); - it('should return additional headers even if not set in ' + - 'put object request', done => { - const params = { - Bucket: bucketName, - Key: objectName, - ResponseCacheControl: cacheControl, - ResponseContentDisposition: contentDisposition, - ResponseContentEncoding: contentEncoding, - ResponseContentLanguage: contentLanguage, - ResponseContentType: contentType, - ResponseExpires: expires, - }; - s3.send(new GetObjectCommand(params)).then(res => { - assert.strictEqual(res.CacheControl, - cacheControl); - assert.strictEqual(res.ContentDisposition, - contentDisposition); - assert.strictEqual(res.ContentEncoding, - contentEncoding); - assert.strictEqual(res.ContentLanguage, - contentLanguage); - assert.strictEqual(res.ContentType, contentType); - assert.strictEqual(res.Expires.toGMTString(), - new Date(expires).toGMTString()); - return done(); - }).catch(done); + it('should return additional headers even if not set in ' + 'put object request', done => { + const params = { + Bucket: bucketName, + Key: objectName, + ResponseCacheControl: cacheControl, + ResponseContentDisposition: contentDisposition, + ResponseContentEncoding: contentEncoding, + ResponseContentLanguage: contentLanguage, + ResponseContentType: contentType, + ResponseExpires: expires, + }; + s3.send(new GetObjectCommand(params)) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + assert.strictEqual(res.ContentEncoding, contentEncoding); + assert.strictEqual(res.ContentLanguage, contentLanguage); + assert.strictEqual(res.ContentType, contentType); + assert.strictEqual(res.Expires.toGMTString(), new Date(expires).toGMTString()); + return done(); + }) + .catch(done); + }); }); - }); - }); + }, + ); describe('x-amz-website-redirect-location header', () => { before(async () => { @@ -329,12 +383,13 @@ describe('GET object', () => { }; await s3.send(new PutObjectCommand(params)); }); - it('should return website redirect header if specified in ' + - 'objectPUT request', done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })).then(res => { - assert.strictEqual(res.WebsiteRedirectLocation, '/'); - return done(); - }).catch(done); + it('should return website redirect header if specified in ' + 'objectPUT request', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.WebsiteRedirectLocation, '/'); + return done(); + }) + .catch(done); }); }); @@ -359,27 +414,31 @@ describe('GET object', () => { await s3.send(new PutObjectCommand(params)); }); - it('should not return "x-amz-tagging-count" if no tag ' + - 'associated with the object', - done => { - s3.send(new GetObjectCommand(params)).then(data => { - assert.strictEqual(data.TagCount, undefined); - return done(); - }).catch(done); + it('should not return "x-amz-tagging-count" if no tag ' + 'associated with the object', done => { + s3.send(new GetObjectCommand(params)) + .then(data => { + assert.strictEqual(data.TagCount, undefined); + return done(); + }) + .catch(done); }); describe('tag associated with the object', () => { beforeEach(async () => { await s3.send(new PutObjectTaggingCommand(paramsTagging)); }); - it('should return "x-amz-tagging-count" header that provides ' + - 'the count of number of tags associated with the object', - done => { - s3.send(new GetObjectCommand(params)).then(data => { - assert.equal(data.TagCount, 1); - return done(); - }).catch(done); - }); + it( + 'should return "x-amz-tagging-count" header that provides ' + + 'the count of number of tags associated with the object', + done => { + s3.send(new GetObjectCommand(params)) + .then(data => { + assert.equal(data.TagCount, 1); + return done(); + }) + .catch(done); + }, + ); }); }); @@ -388,43 +447,33 @@ describe('GET object', () => { beforeEach(async () => { await s3.send(new PutObjectCommand(params)); }); - it('If-Match: returns no error when ETag match, with double ' + - 'quotes around ETag', - done => { - requestGet({ IfMatch: etag }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, with double ' + 'quotes around ETag', done => { + requestGet({ IfMatch: etag }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, with ' + - 'double quotes around ETag', - done => { - requestGet({ IfMatch: - `non-matching,${etag}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, with ' + 'double quotes around ETag', done => { + requestGet({ IfMatch: `non-matching,${etag}` }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', - done => { - requestGet({ IfMatch: etagTrim }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', done => { + requestGet({ IfMatch: etagTrim }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', - done => { - requestGet({ IfMatch: - `non-matching,${etagTrim}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', done => { + requestGet({ IfMatch: `non-matching,${etagTrim}` }, err => { + checkNoError(err); + done(); }); + }); it('If-Match: returns no error when ETag match with *', done => { requestGet({ IfMatch: '*' }, err => { @@ -433,320 +482,364 @@ describe('GET object', () => { }); }); - it('If-Match: returns PreconditionFailed when ETag does not match', - done => { - requestGet({ + it('If-Match: returns PreconditionFailed when ETag does not match', done => { + requestGet( + { IfMatch: 'non-matching ETag', - }, err => { + }, + err => { checkError(err, 'PreconditionFailed'); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns no error when ETag does not match', - done => { - requestGet({ IfNoneMatch: 'non-matching' }, err => { - checkNoError(err); - done(); - }); + it('If-None-Match: returns no error when ETag does not match', done => { + requestGet({ IfNoneMatch: 'non-matching' }, err => { + checkNoError(err); + done(); }); + }); - it('If-None-Match: returns no error when all ETags do not match', - done => { - requestGet({ - IfNoneMatch: 'non-matching,' + - 'non-matching-either', - }, err => { + it('If-None-Match: returns no error when all ETags do not match', done => { + requestGet( + { + IfNoneMatch: 'non-matching,' + 'non-matching-either', + }, + err => { checkNoError(err); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when ETag match, with ' + - 'double quotes around ETag', - done => { - requestGet({ IfNoneMatch: etag }, err => { - checkError(err, 'NotModified'); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, with ' + 'double quotes around ETag', done => { + requestGet({ IfNoneMatch: etag }, err => { + checkError(err, 'NotModified'); + done(); }); + }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'with double quotes around ETag', + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'with double quotes around ETag', done => { - requestGet({ - IfNoneMatch: `non-matching,${etag}`, - }, err => { - checkError(err, 'NotModified'); - done(); - }); - }); + requestGet( + { + IfNoneMatch: `non-matching,${etag}`, + }, + err => { + checkError(err, 'NotModified'); + done(); + }, + ); + }, + ); - it('If-None-Match: returns NotModified when value is "*"', - done => { - requestGet({ + it('If-None-Match: returns NotModified when value is "*"', done => { + requestGet( + { IfNoneMatch: '*', - }, err => { - checkError(err, 'NotModified'); - done(); - }); - }); - - it('If-None-Match: returns NotModified when ETag match, without ' + - 'double quotes around ETag', - done => { - requestGet({ IfNoneMatch: etagTrim }, err => { + }, + err => { checkError(err, 'NotModified'); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'without double quotes around ETag', - done => { - requestGet({ - IfNoneMatch: `non-matching,${etagTrim}`, - }, err => { - checkError(err, 'NotModified'); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, without ' + 'double quotes around ETag', done => { + requestGet({ IfNoneMatch: etagTrim }, err => { + checkError(err, 'NotModified'); + done(); }); + }); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'without double quotes around ETag', done => { - requestGet({ IfModifiedSince: dateFromNow(-1) }, + requestGet( + { + IfNoneMatch: `non-matching,${etagTrim}`, + }, err => { - checkNoError(err); + checkError(err, 'NotModified'); done(); - }); + }, + ); + }, + ); + + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', done => { + requestGet({ IfModifiedSince: dateFromNow(-1) }, err => { + checkNoError(err); + done(); }); + }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns NotModified if Last modified ' + - 'date is lesser', - done => { - requestGet({ IfModifiedSince: dateFromNow(1) }, - err => { - checkError(err, 'NotModified'); - done(); - }); + it.skip('If-Modified-Since: returns NotModified if Last modified ' + 'date is lesser', done => { + requestGet({ IfModifiedSince: dateFromNow(1) }, err => { + checkError(err, 'NotModified'); + done(); }); + }); - it('If-Modified-Since: returns NotModified if Last modified ' + - 'date is equal', - done => { - s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })).then(data => { + it('If-Modified-Since: returns NotModified if Last modified ' + 'date is equal', done => { + s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(data => { const lastModified = dateConvert(data.LastModified); requestGet({ IfModifiedSince: lastModified }, err => { checkError(err, 'NotModified'); done(); }); - }).catch(done); - }); + }) + .catch(done); + }); - it('If-Unmodified-Since: returns no error when lastModified date ' + - 'is greater', - done => { - requestGet({ IfUnmodifiedSince: dateFromNow(1) }, - err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified date ' + 'is greater', done => { + requestGet({ IfUnmodifiedSince: dateFromNow(1) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', done => { - s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })).then(data => { - const lastModified = dateConvert(data.LastModified); - requestGet({ IfUnmodifiedSince: lastModified }, - err => { + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', done => { + s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(data => { + const lastModified = dateConvert(data.LastModified); + requestGet({ IfUnmodifiedSince: lastModified }, err => { checkNoError(err); done(); }); - }).catch(done); + }) + .catch(done); }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', - done => { - requestGet({ IfUnmodifiedSince: dateFromNow(-1) }, - err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', done => { + requestGet({ IfUnmodifiedSince: dateFromNow(-1) }, err => { + checkError(err, 'PreconditionFailed'); + done(); }); + }); - it('If-Match & If-Unmodified-Since: returns no error when match ' + - 'Etag and lastModified is greater', + it( + 'If-Match & If-Unmodified-Since: returns no error when match ' + 'Etag and lastModified is greater', done => { - requestGet({ + requestGet( + { + IfMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); + }, + ); + + it('If-Match match & If-Unmodified-Since match', done => { + requestGet( + { IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { + IfUnmodifiedSince: dateFromNow(1), + }, + err => { checkNoError(err); done(); - }); - }); - - it('If-Match match & If-Unmodified-Since match', done => { - requestGet({ - IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + }, + ); }); it('If-Match not match & If-Unmodified-Since not match', done => { - requestGet({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-Match not match & If-Unmodified-Since match', done => { - requestGet({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-Match match & If-Modified-Since not match', done => { - requestGet({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match match & If-Modified-Since match', done => { - requestGet({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since not match', done => { - requestGet({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since match', done => { - requestGet({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); - it('If-None-Match & If-Modified-Since: returns NotModified when ' + - 'Etag does not match and lastModified is greater', + it( + 'If-None-Match & If-Modified-Since: returns NotModified when ' + + 'Etag does not match and lastModified is greater', done => { - requestGet({ + requestGet( + { + IfNoneMatch: etagTrim, + IfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'NotModified'); + done(); + }, + ); + }, + ); + + it('If-None-Match not match & If-Modified-Since not match', done => { + requestGet( + { IfNoneMatch: etagTrim, IfModifiedSince: dateFromNow(1), - }, err => { + }, + err => { checkError(err, 'NotModified'); done(); - }); - }); - - it('If-None-Match not match & If-Modified-Since not match', - done => { - requestGet({ - IfNoneMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'NotModified'); - done(); - }); + }, + ); }); it('If-None-Match match & If-Modified-Since match', done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-None-Match match & If-Modified-Since not match', - done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + it.skip('If-None-Match match & If-Modified-Since not match', done => { + requestGet( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since match', done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestGet( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since not match', done => { - requestGet({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + requestGet( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since match', done => { - requestGet({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkError(err, 'NotModified'); - done(); - }); + requestGet( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkError(err, 'NotModified'); + done(); + }, + ); }); - it('If-None-Match not match & If-Unmodified-Since not match', - done => { - requestGet({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - checkError(err, 'PreconditionFailed'); - done(); - }); + it('If-None-Match not match & If-Unmodified-Since not match', done => { + requestGet( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkError(err, 'PreconditionFailed'); + done(); + }, + ); }); }); @@ -756,8 +849,8 @@ describe('GET object', () => { const invalidPartNumbers = [-1, 0, 10001]; orderedPartNumbers.forEach(num => - it(`should get the body of part ${num} when ordered MPU`, - done => completeMPU(orderedPartNumbers, err => { + it(`should get the body of part ${num} when ordered MPU`, done => + completeMPU(orderedPartNumbers, err => { checkNoError(err); return requestGet({ PartNumber: num }, async (err, data) => { checkNoError(err); @@ -768,135 +861,143 @@ describe('GET object', () => { const bodyText = await data.Body.transformToString(); assert.strictEqual( md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') + md5HashExpected.update(expected).digest('hex'), ); return done(); }); - }))); + })), + ); // Use the orderedPartNumbers to retrieve parts with GetObject. orderedPartNumbers.forEach(num => - it(`should get the body of part ${num} when unordered MPU`, - done => completeMPU(unOrderedPartNumbers, err => { + it(`should get the body of part ${num} when unordered MPU`, done => + completeMPU(unOrderedPartNumbers, err => { checkNoError(err); return requestGet({ PartNumber: num }, async (err, data) => { checkNoError(err); checkIntegerHeader(data.ContentLength, partSize); const md5Hash = crypto.createHash('md5'); const md5HashExpected = crypto.createHash('md5'); - const expected = Buffer.alloc(partSize) - .fill(unOrderedPartNumbers[num - 1]); + const expected = Buffer.alloc(partSize).fill(unOrderedPartNumbers[num - 1]); const bodyText = await data.Body.transformToString(); assert.strictEqual( md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') + md5HashExpected.update(expected).digest('hex'), ); return done(); }); - }))); + })), + ); invalidPartNumbers.forEach(num => - it(`should not accept a partNumber that is not 1-10000: ${num}`, - done => completeMPU(orderedPartNumbers, err => { - checkNoError(err); - return requestGet({ PartNumber: num }, err => { - checkError(err, 'InvalidArgument'); - done(); - }); - }))); + it(`should not accept a partNumber that is not 1-10000: ${num}`, done => + completeMPU(orderedPartNumbers, err => { + checkNoError(err); + return requestGet({ PartNumber: num }, err => { + checkError(err, 'InvalidArgument'); + done(); + }); + })), + ); - it('should not accept a part number greater than the total parts ' + - 'uploaded for an MPU', done => + it('should not accept a part number greater than the total parts ' + 'uploaded for an MPU', done => completeMPU(orderedPartNumbers, err => { checkNoError(err); return requestGet({ PartNumber: 11 }, err => { checkError(err, 'InvalidPartNumber'); done(); }); - })); + }), + ); - it('should accept a part number of 1 for regular put object', - async () => { - await s3.send(new PutObjectCommand({ + it('should accept a part number of 1 for regular put object', async () => { + await s3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectName, Body: Buffer.alloc(10), - })); - - const data = await requestGetPromise({ PartNumber: 1 }); - const md5Hash = crypto.createHash('md5'); - const md5HashExpected = crypto.createHash('md5'); - const expected = Buffer.alloc(10).fill(0); - const bodyText = await data.Body.transformToString(); - assert.strictEqual( - md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') - ); - }); + }), + ); + + const data = await requestGetPromise({ PartNumber: 1 }); + const md5Hash = crypto.createHash('md5'); + const md5HashExpected = crypto.createHash('md5'); + const expected = Buffer.alloc(10).fill(0); + const bodyText = await data.Body.transformToString(); + assert.strictEqual( + md5Hash.update(bodyText).digest('hex'), + md5HashExpected.update(expected).digest('hex'), + ); + }); it('should accept a part number that is a string', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: Buffer.alloc(10), - })); - + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(10), + }), + ); + const data = await requestGetPromise({ PartNumber: '1' }); - checkIntegerHeader(data.ContentLength, 10); + checkIntegerHeader(data.ContentLength, 10); const md5Hash = crypto.createHash('md5'); const md5HashExpected = crypto.createHash('md5'); const expected = Buffer.alloc(10).fill(0); const bodyText = await data.Body.transformToString(); assert.strictEqual( md5Hash.update(bodyText).digest('hex'), - md5HashExpected.update(expected).digest('hex') + md5HashExpected.update(expected).digest('hex'), ); }); - it('should not accept a part number greater than 1 for regular ' + - 'put object', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: Buffer.alloc(10), - })); - + it('should not accept a part number greater than 1 for regular ' + 'put object', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(10), + }), + ); + await assert.rejects( - () => requestGetPromise({ PartNumber: 2 }), - err => { - checkError(err, 'InvalidPartNumber'); - return true; - } - ); + () => requestGetPromise({ PartNumber: 2 }), + err => { + checkError(err, 'InvalidPartNumber'); + return true; + }, + ); }); it('should not accept both PartNumber and Range as params', done => completeMPU(orderedPartNumbers, err => { checkNoError(err); - return requestGet({ - PartNumber: 1, - Range: 'bytes=0-10', - }, err => { - checkError(err, 'InvalidRequest'); - done(); - }); + return requestGet( + { + PartNumber: 1, + Range: 'bytes=0-10', + }, + err => { + checkError(err, 'InvalidRequest'); + done(); + }, + ); })); - it('should not include PartsCount response header for regular ' + - 'put object', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: Buffer.alloc(10), - })); - + it('should not include PartsCount response header for regular ' + 'put object', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: Buffer.alloc(10), + }), + ); + const data = await requestGetPromise({ PartNumber: 1 }); - assert.strictEqual('PartsCount' in data, false, - 'PartsCount header is present.'); + assert.strictEqual('PartsCount' in data, false, 'PartsCount header is present.'); }); - it('should include PartsCount response header for mpu object', - done => { + it('should include PartsCount response header for mpu object', done => { completeMPU(orderedPartNumbers, err => { assert.ifError(err); return requestGet({ PartNumber: 1 }, (err, data) => { @@ -910,123 +1011,151 @@ describe('GET object', () => { describe('uploadPartCopy', () => { // The original object was composed of three parts const partOneSize = partSize * 10; - const bufs = orderedPartNumbers.map(n => - Buffer.alloc(partSize, n)); + const bufs = orderedPartNumbers.map(n => Buffer.alloc(partSize, n)); const partOneBody = Buffer.concat(bufs, partOneSize); const partTwoBody = Buffer.alloc(partSize, 4); - beforeEach(done => async.waterfall([ - next => completeMPU(orderedPartNumbers, next), - next => createMPUAndPutTwoParts(partTwoBody, next), - (uploadId, ETags, next) => - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - MultipartUpload: { - Parts: [ - { - ETag: ETags[0], - PartNumber: 1, - }, - { - ETag: ETags[1], - PartNumber: 2, - }, - ], - }, - UploadId: uploadId, - })).then(() => next()).catch(next), - ], done)); + beforeEach(done => + async.waterfall( + [ + next => completeMPU(orderedPartNumbers, next), + next => createMPUAndPutTwoParts(partTwoBody, next), + (uploadId, ETags, next) => + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + MultipartUpload: { + Parts: [ + { + ETag: ETags[0], + PartNumber: 1, + }, + { + ETag: ETags[1], + PartNumber: 2, + }, + ], + }, + UploadId: uploadId, + }), + ) + .then(() => next()) + .catch(next), + ], + done, + ), + ); afterEach(async () => { - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: copyPartKey, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: copyPartKey, + }), + ); }); it('should retrieve a part copied from an MPU', done => - checkGetObjectPart(copyPartKey, 1, partOneSize, partOneBody, - done)); + checkGetObjectPart(copyPartKey, 1, partOneSize, partOneBody, done)); - it('should retrieve a part put after part copied from MPU', - done => checkGetObjectPart(copyPartKey, 2, partSize, - partTwoBody, done)); + it('should retrieve a part put after part copied from MPU', done => + checkGetObjectPart(copyPartKey, 2, partSize, partTwoBody, done)); }); describe('uploadPartCopy overwrite', () => { const partOneBody = Buffer.alloc(partSize, 1); // The original object was composed of three parts const partTwoSize = partSize * 10; - const bufs = orderedPartNumbers.map(n => - Buffer.alloc(partSize, n)); + const bufs = orderedPartNumbers.map(n => Buffer.alloc(partSize, n)); const partTwoBody = Buffer.concat(bufs, partTwoSize); - beforeEach(done => async.waterfall([ - next => completeMPU(orderedPartNumbers, next), - next => createMPUAndPutTwoParts(partTwoBody, next), - /* eslint-disable no-param-reassign */ - // Overwrite part one. - (uploadId, ETags, next) => - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: copyPartKey, - PartNumber: 1, - UploadId: uploadId, - Body: partOneBody, - })).then(data => { - ETags[0] = data.ETag; - return next(null, uploadId, ETags); - }).catch(next), - // Overwrite part one with an three-part object. - (uploadId, ETags, next) => - s3.send(new UploadPartCopyCommand({ - Bucket: bucketName, - CopySource: `/${bucketName}/${objectName}`, - Key: copyPartKey, - PartNumber: 2, - UploadId: uploadId, - })).then(data => { - ETags[1] = data.CopyPartResult.ETag; - return next(null, uploadId, ETags); - }).catch(next), - /* eslint-enable no-param-reassign */ - (uploadId, ETags, next) => - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: copyPartKey, - MultipartUpload: { - Parts: [ - { - ETag: ETags[0], - PartNumber: 1, - }, - { - ETag: ETags[1], - PartNumber: 2, - }, - ], - }, - UploadId: uploadId, - })).then(() => next()).catch(next), - ], done)); + beforeEach(done => + async.waterfall( + [ + next => completeMPU(orderedPartNumbers, next), + next => createMPUAndPutTwoParts(partTwoBody, next), + /* eslint-disable no-param-reassign */ + // Overwrite part one. + (uploadId, ETags, next) => + s3 + .send( + new UploadPartCommand({ + Bucket: bucketName, + Key: copyPartKey, + PartNumber: 1, + UploadId: uploadId, + Body: partOneBody, + }), + ) + .then(data => { + ETags[0] = data.ETag; + return next(null, uploadId, ETags); + }) + .catch(next), + // Overwrite part one with an three-part object. + (uploadId, ETags, next) => + s3 + .send( + new UploadPartCopyCommand({ + Bucket: bucketName, + CopySource: `/${bucketName}/${objectName}`, + Key: copyPartKey, + PartNumber: 2, + UploadId: uploadId, + }), + ) + .then(data => { + ETags[1] = data.CopyPartResult.ETag; + return next(null, uploadId, ETags); + }) + .catch(next), + /* eslint-enable no-param-reassign */ + (uploadId, ETags, next) => + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: copyPartKey, + MultipartUpload: { + Parts: [ + { + ETag: ETags[0], + PartNumber: 1, + }, + { + ETag: ETags[1], + PartNumber: 2, + }, + ], + }, + UploadId: uploadId, + }), + ) + .then(() => next()) + .catch(next), + ], + done, + ), + ); afterEach(async () => { - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: copyPartKey, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: copyPartKey, + }), + ); }); - it('should retrieve a part that overwrote another part ' + - 'originally copied from an MPU', done => - checkGetObjectPart(copyPartKey, 1, partSize, partOneBody, - done)); + it('should retrieve a part that overwrote another part ' + 'originally copied from an MPU', done => + checkGetObjectPart(copyPartKey, 1, partSize, partOneBody, done), + ); - it('should retrieve a part copied from an MPU after the ' + - 'original part was overwritten', - done => checkGetObjectPart(copyPartKey, 2, partTwoSize, - partTwoBody, done)); + it('should retrieve a part copied from an MPU after the ' + 'original part was overwritten', done => + checkGetObjectPart(copyPartKey, 2, partTwoSize, partTwoBody, done), + ); }); }); @@ -1038,19 +1167,18 @@ describe('GET object', () => { }; await s3.send(new PutObjectCommand(params)); }); - it('should return website redirect header if specified in ' + - 'objectPUT request', done => { - s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })).then(res => { - assert.strictEqual(res.WebsiteRedirectLocation, - undefined); - return done(); - }).catch(done); + it('should return website redirect header if specified in ' + 'objectPUT request', done => { + s3.send(new GetObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.WebsiteRedirectLocation, undefined); + return done(); + }) + .catch(done); }); }); }); }); - describe('GET object with object lock', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -1070,59 +1198,69 @@ describe('GET object with object lock', () => { ObjectLockMode: mockMode, ObjectLockLegalHoldStatus: 'ON', }; - return s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })) - .then(() => s3.send(new PutObjectCommand(params))) - .then(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }))) - /* eslint-disable no-return-assign */ - .then(res => versionId = res.VersionId) - .catch(err => { - process.stdout.write('Error in before\n'); - throw err; - }); + return ( + s3 + .send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => s3.send(new PutObjectCommand(params))) + .then(() => s3.send(new GetObjectCommand({ Bucket: bucket, Key: key }))) + /* eslint-disable no-return-assign */ + .then(res => (versionId = res.VersionId)) + .catch(err => { + process.stdout.write('Error in before\n'); + throw err; + }) + ); }); - afterEach(() => changeLockPromise([{ bucket, key, versionId }], '') - .then(() => s3.send(new ListObjectVersionsCommand({ Bucket: bucket }))) - .then(res => res.Versions?.forEach(object => { - const params = [ - { - bucket, - key: object.Key, - versionId: object.VersionId, - }, - ]; - changeLockPromise(params, ''); - })) - .then(() => { - process.stdout.write('Emptying and deleting buckets\n'); - return bucketUtil.empty(bucket); - }) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - })); + afterEach(() => + changeLockPromise([{ bucket, key, versionId }], '') + .then(() => s3.send(new ListObjectVersionsCommand({ Bucket: bucket }))) + .then(res => + res.Versions?.forEach(object => { + const params = [ + { + bucket, + key: object.Key, + versionId: object.VersionId, + }, + ]; + changeLockPromise(params, ''); + }), + ) + .then(() => { + process.stdout.write('Emptying and deleting buckets\n'); + return bucketUtil.empty(bucket); + }) + .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }), + ); it('should return object lock headers if set on the object', done => { - s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })).then(res => { - assert.strictEqual(res.ObjectLockMode, mockMode); - const responseDate - = formatDate(res.ObjectLockRetainUntilDate); - const expectedDate = formatDate(mockDate); - assert.strictEqual(responseDate, expectedDate); - assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); - const objectWithLock = [ - { - bucket, - key, - versionId: res.VersionId, - }, - ]; - changeObjectLock(objectWithLock, '', done); - }).catch(done); + s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + .then(res => { + assert.strictEqual(res.ObjectLockMode, mockMode); + const responseDate = formatDate(res.ObjectLockRetainUntilDate); + const expectedDate = formatDate(mockDate); + assert.strictEqual(responseDate, expectedDate); + assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); + const objectWithLock = [ + { + bucket, + key, + versionId: res.VersionId, + }, + ]; + changeObjectLock(objectWithLock, '', done); + }) + .catch(done); }); }); }); @@ -1138,10 +1276,10 @@ describe('GET object checksum mode', () => { const expectedDigests = {}; const checksumAlgorithms = [ - { algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' }, - { algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' }, - { algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' }, - { algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' }, + { algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' }, + { algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' }, + { algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' }, + { algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' }, { algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' }, ]; @@ -1149,14 +1287,12 @@ describe('GET object checksum mode', () => { // Disable automatic response checksum validation so the SDK does // not silently add x-amz-checksum-mode: ENABLED to every GetObject // request, which would interfere with the "mode not set" test. - bucketUtil = new BucketUtility('default', - { ...sigCfg, responseChecksumValidation: 'WHEN_REQUIRED' }); + bucketUtil = new BucketUtility('default', { ...sigCfg, responseChecksumValidation: 'WHEN_REQUIRED' }); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: checksumBucket })); for (const { internalName } of checksumAlgorithms) { - expectedDigests[internalName] = - await algorithms[internalName].digest(body); + expectedDigests[internalName] = await algorithms[internalName].digest(body); } }); @@ -1167,56 +1303,71 @@ describe('GET object checksum mode', () => { checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => { it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - Body: body, - ChecksumAlgorithm: algorithm, - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + Body: body, + ChecksumAlgorithm: algorithm, + }), + ); const storedChecksum = putRes[responseField]; assert(storedChecksum, `Expected ${responseField} in PutObject response`); - const getRes = await s3.send(new GetObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - ChecksumMode: 'ENABLED', - })); - assert.strictEqual(getRes[responseField], expectedDigests[internalName], - `${responseField} value mismatch`); + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + ChecksumMode: 'ENABLED', + }), + ); + assert.strictEqual( + getRes[responseField], + expectedDigests[internalName], + `${responseField} value mismatch`, + ); assert.strictEqual(getRes[responseField], storedChecksum); assert.strictEqual(getRes.ChecksumType, 'FULL_OBJECT'); }); }); it('should not return checksum headers when ChecksumMode is not set', async () => { - await s3.send(new PutObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - Body: body, - ChecksumAlgorithm: 'SHA256', - })); - - const getRes = await s3.send(new GetObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + Body: body, + ChecksumAlgorithm: 'SHA256', + }), + ); + + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + }), + ); assert.strictEqual(getRes.ChecksumSHA256, undefined); assert.strictEqual(getRes.ChecksumType, undefined); }); it('should return an error when ChecksumMode is not ENABLED', async () => { - await s3.send(new PutObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - Body: body, - })); - - await assert.rejects( - s3.send(new GetObjectCommand({ + await s3.send( + new PutObjectCommand({ Bucket: checksumBucket, Key: checksumKey, - ChecksumMode: 'DISABLED', - })), + Body: body, + }), + ); + + await assert.rejects( + s3.send( + new GetObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + ChecksumMode: 'DISABLED', + }), + ), err => { assert.strictEqual(err.name, 'InvalidArgument'); return true; diff --git a/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js b/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js index 22d1636c0d..31cf3efb78 100644 --- a/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/getMPU_compatibleHeaders.js @@ -13,114 +13,122 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const bucketName = 'testgetmpubucket'; const objectName = 'key'; -describe('GET multipart upload object [Cache-Control, Content-Disposition, ' + -'Content-Encoding, Expires headers]', () => { - withV4(sigCfg => { - let bucketUtil; - let s3; - let uploadId; - const cacheControl = 'max-age=86400'; - const contentDisposition = 'attachment; filename="fname.ext";'; - const contentEncoding = 'aws-chunked,gzip'; - // AWS Node SDK requires Date object, ISO-8601 string, or - // a UNIX timestamp for Expires header - const expires = new Date(); +describe( + 'GET multipart upload object [Cache-Control, Content-Disposition, ' + 'Content-Encoding, Expires headers]', + () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + let uploadId; + const cacheControl = 'max-age=86400'; + const contentDisposition = 'attachment; filename="fname.ext";'; + const contentEncoding = 'aws-chunked,gzip'; + // AWS Node SDK requires Date object, ISO-8601 string, or + // a UNIX timestamp for Expires header + const expires = new Date(); - before(() => { - const params = { - Bucket: bucketName, - Key: objectName, - CacheControl: cacheControl, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - Expires: expires, - }; - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return bucketUtil.empty(bucketName) - .then(() => { - process.stdout.write('deleting bucket, just in case\n'); - return bucketUtil.deleteOne(bucketName); - }) - .catch(err => { - if (err.name !== 'NoSuchBucket') { - process.stdout.write(`${err}\n`); - throw err; - } - }) - .then(() => { - process.stdout.write('creating bucket\n'); - return s3.send(new CreateBucketCommand({ Bucket: bucketName })); - }) - .then(() => { - process.stdout.write('initiating multipart upload\n'); - return s3.send(new CreateMultipartUploadCommand(params)); - }) - .then(res => { - uploadId = res.UploadId; - return uploadId; - }) - .catch(err => { - process.stdout.write(`Error in before: ${err}\n`); - throw err; - }); - }); - after(() => { - process.stdout.write('Emptying bucket\n'); - return bucketUtil.empty(bucketName) - .then(() => { - process.stdout.write('Deleting bucket\n'); - return bucketUtil.deleteOne(bucketName); - }) - .catch(err => { - process.stdout.write('Error in after\n'); - throw err; - }); - }); - it('should return additional headers when get request is performed ' + - 'on MPU, when they are specified in creation of MPU', - () => { - const params = { Bucket: bucketName, Key: 'key', PartNumber: 1, - UploadId: uploadId }; - return s3.send(new UploadPartCommand(params)) - .catch(err => { - process.stdout.write(`Error in uploadPart ${err}\n`); - throw err; - }) - .then(res => { - process.stdout.write('about to complete multipart upload\n'); - return s3.send(new CompleteMultipartUploadCommand({ + before(() => { + const params = { Bucket: bucketName, Key: objectName, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { ETag: res.ETag, PartNumber: 1 }, - ], - }, - })); - }) - .catch(err => { - process.stdout.write(`Error completing upload ${err}\n`); - throw err; - }) - .then(() => { - process.stdout.write('about to get object\n'); - return s3.send(new GetObjectCommand({ - Bucket: bucketName, Key: objectName, - })); - }) - .catch(err => { - process.stdout.write(`Error getting object ${err}\n`); - throw err; - }) - .then(res => { - assert.strictEqual(res.CacheControl, cacheControl); - assert.strictEqual(res.ContentDisposition, contentDisposition); - assert.strictEqual(res.ContentEncoding, 'gzip'); - assert.strictEqual(res.Expires.toGMTString(), - expires.toGMTString()); + CacheControl: cacheControl, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + Expires: expires, + }; + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return bucketUtil + .empty(bucketName) + .then(() => { + process.stdout.write('deleting bucket, just in case\n'); + return bucketUtil.deleteOne(bucketName); + }) + .catch(err => { + if (err.name !== 'NoSuchBucket') { + process.stdout.write(`${err}\n`); + throw err; + } + }) + .then(() => { + process.stdout.write('creating bucket\n'); + return s3.send(new CreateBucketCommand({ Bucket: bucketName })); + }) + .then(() => { + process.stdout.write('initiating multipart upload\n'); + return s3.send(new CreateMultipartUploadCommand(params)); + }) + .then(res => { + uploadId = res.UploadId; + return uploadId; + }) + .catch(err => { + process.stdout.write(`Error in before: ${err}\n`); + throw err; + }); + }); + after(() => { + process.stdout.write('Emptying bucket\n'); + return bucketUtil + .empty(bucketName) + .then(() => { + process.stdout.write('Deleting bucket\n'); + return bucketUtil.deleteOne(bucketName); + }) + .catch(err => { + process.stdout.write('Error in after\n'); + throw err; + }); }); + it( + 'should return additional headers when get request is performed ' + + 'on MPU, when they are specified in creation of MPU', + () => { + const params = { Bucket: bucketName, Key: 'key', PartNumber: 1, UploadId: uploadId }; + return s3 + .send(new UploadPartCommand(params)) + .catch(err => { + process.stdout.write(`Error in uploadPart ${err}\n`); + throw err; + }) + .then(res => { + process.stdout.write('about to complete multipart upload\n'); + return s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: objectName, + UploadId: uploadId, + MultipartUpload: { + Parts: [{ ETag: res.ETag, PartNumber: 1 }], + }, + }), + ); + }) + .catch(err => { + process.stdout.write(`Error completing upload ${err}\n`); + throw err; + }) + .then(() => { + process.stdout.write('about to get object\n'); + return s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + }) + .catch(err => { + process.stdout.write(`Error getting object ${err}\n`); + throw err; + }) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + assert.strictEqual(res.ContentEncoding, 'gzip'); + assert.strictEqual(res.Expires.toGMTString(), expires.toGMTString()); + }); + }, + ); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/object/getObjTagging.js b/tests/functional/aws-node-sdk/test/object/getObjTagging.js index 282866e7db..1bfbfc20c9 100644 --- a/tests/functional/aws-node-sdk/test/object/getObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/getObjTagging.js @@ -16,16 +16,18 @@ const bucketName = 'testtaggingbucket'; const objectName = 'testtaggingobject'; const objectNameAcl = 'testtaggingobjectacl'; -const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }, - { - Key: 'key2', - Value: 'value2', - }, -] }; +const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + { + Key: 'key2', + Value: 'value2', + }, + ], +}; describe('GET object taggings', () => { withV4(sigCfg => { @@ -41,140 +43,174 @@ describe('GET object taggings', () => { afterEach(() => { process.stdout.write('Emptying bucket'); - return bucketUtil.empty(bucketName) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucketName); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .empty(bucketName) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucketName); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); it('should return appropriate tags after putting tags', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); }); it('should return no tag after putting and deleting tags', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, []); }); - it('should return empty array after putting no tag', - async () => { - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + it('should return empty array after putting no tag', async () => { + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, []); }); - it('should return NoSuchKey getting tag set to a non-existing object', - async () => { + it('should return NoSuchKey getting tag set to a non-existing object', async () => { try { - await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: 'nonexisting', - })); + await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: 'nonexisting', + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { checkError(err, 'NoSuchKey', 404); } }); - it('should return 403 AccessDenied getting tag set with another ' + - 'account', async () => { + it('should return 403 AccessDenied getting tag set with another ' + 'account', async () => { try { - await otherAccountS3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + await otherAccountS3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { checkError(err, 'AccessDenied', 403); } }); - it('should return 403 AccessDenied getting tag with a different ' + - 'account to an object with ACL "public-read-write"', - async () => { - try { - await s3.send(new PutBucketAclCommand({ + it( + 'should return 403 AccessDenied getting tag with a different ' + + 'account to an object with ACL "public-read-write"', + async () => { + try { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); + + it( + 'should return 403 AccessDenied getting tag set to an object' + + ' in a bucket created with a different account', + async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + + try { + await otherAccountS3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); + + it('should get tag to an object in a bucket created with same ' + 'account', async () => { + await s3.send( + new PutBucketAclCommand({ Bucket: bucketName, ACL: 'public-read-write', - })); - await otherAccountS3.send(new GetObjectTaggingCommand({ + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ Bucket: bucketName, - Key: objectName, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); - - it('should return 403 AccessDenied getting tag set to an object' + - ' in a bucket created with a different account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + Key: objectNameAcl, + }), + ); - try { - await otherAccountS3.send(new GetObjectTaggingCommand({ + const data = await s3.send( + new GetObjectTaggingCommand({ Bucket: bucketName, Key: objectNameAcl, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); - - it('should get tag to an object in a bucket created with same ' + - 'account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); - - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + }), + ); assert.deepStrictEqual(data.TagSet, []); }); diff --git a/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js b/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js index 1424897ced..b40d9551c9 100644 --- a/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js +++ b/tests/functional/aws-node-sdk/test/object/getObjectLegalHold.js @@ -20,7 +20,6 @@ const unlockedBucket = 'mock-bucket-no-lock'; const key = 'mock-object-legalhold'; const keyNoHold = 'mock-object-no-legalhold'; - describe('GET object legal hold', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -32,10 +31,12 @@ describe('GET object legal hold', () => { beforeEach(async () => { process.stdout.write('Putting buckets and objects\n'); process.stdout.write('Putting object legal hold\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: key })); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: keyNoHold })); @@ -43,99 +44,141 @@ describe('GET object legal hold', () => { const res = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); versionId = res.VersionId; process.stdout.write('Putting object legal hold\n'); - await s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - LegalHold: { Status: 'ON' }, - })); + await s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + LegalHold: { Status: 'ON' }, + }), + ); }); afterEach(() => { process.stdout.write('Removing object lock\n'); return changeLockPromise([{ bucket, key, versionId }], {}) - .then(() => { - process.stdout.write('Emptying and deleting buckets\n'); - return bucketUtil.empty(bucket); - }) - .then(() => bucketUtil.empty(unlockedBucket)) - .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + .then(() => { + process.stdout.write('Emptying and deleting buckets\n'); + return bucketUtil.empty(bucket); + }) + .then(() => bucketUtil.empty(unlockedBucket)) + .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); - it('should return AccessDenied getting legal hold with another account', - () => otherAccountS3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - })).then(() => { - throw new Error('Expected AccessDenied error'); - }).catch(err => { - checkError(err, 'AccessDenied', 403); - }) - ); + it('should return AccessDenied getting legal hold with another account', () => + otherAccountS3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => { + throw new Error('Expected AccessDenied error'); + }) + .catch(err => { + checkError(err, 'AccessDenied', 403); + })); - it('should return MethodNotAllowed if object version is delete marker', () => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })).then(res => s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - VersionId: res.VersionId, - })).then(() => { - throw new Error('Expected NoSuchKey error'); - }).catch(err => { - checkError(err, 'MethodNotAllowed', 405); - })).catch(err => { - assert.ifError(err); - }) - ); + it('should return MethodNotAllowed if object version is delete marker', () => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(res => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + VersionId: res.VersionId, + }), + ) + .then(() => { + throw new Error('Expected NoSuchKey error'); + }) + .catch(err => { + checkError(err, 'MethodNotAllowed', 405); + }), + ) + .catch(err => { + assert.ifError(err); + })); - it('should return NoSuchKey if latest version is delete marker', () => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })).then(() => s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - })).then(() => { - throw new Error('Expected NoSuchKey error'); - }).catch(err => { - checkError(err, 'NoSuchKey', 404); - }) - ).catch(err => { - assert.ifError(err); - }) - ); + it('should return NoSuchKey if latest version is delete marker', () => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => { + throw new Error('Expected NoSuchKey error'); + }) + .catch(err => { + checkError(err, 'NoSuchKey', 404); + }), + ) + .catch(err => { + assert.ifError(err); + })); - it('should return InvalidRequest error getting legal hold of object ' + - 'inside object lock disabled bucket', () => s3.send(new GetObjectLegalHoldCommand({ - Bucket: unlockedBucket, - Key: key, - })).then(() => { - throw new Error('Expected InvalidRequest error'); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - }) + it( + 'should return InvalidRequest error getting legal hold of object ' + 'inside object lock disabled bucket', + () => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: unlockedBucket, + Key: key, + }), + ) + .then(() => { + throw new Error('Expected InvalidRequest error'); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + }), ); - it('should return NoSuchObjectLockConfiguration if no legal hold set', () => - s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: keyNoHold, - })).then(() => { - throw new Error('Expected NoSuchObjectLockConfiguration error'); - }).catch(err => { - checkError(err, 'NoSuchObjectLockConfiguration', 404); - }) - ); + it('should return NoSuchObjectLockConfiguration if no legal hold set', () => + s3 + .send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: keyNoHold, + }), + ) + .then(() => { + throw new Error('Expected NoSuchObjectLockConfiguration error'); + }) + .catch(err => { + checkError(err, 'NoSuchObjectLockConfiguration', 404); + })); + + it('should get object legal hold', async () => { + const res = await s3.send( + new GetObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + }), + ); - it('should get object legal hold', async () => { - const res = await s3.send(new GetObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - })); - assert.deepStrictEqual(res.LegalHold, { Status: 'ON' }); await changeLockPromise([{ bucket, key, versionId }], {}); }); diff --git a/tests/functional/aws-node-sdk/test/object/getPartSize.js b/tests/functional/aws-node-sdk/test/object/getPartSize.js index fe9cc4924e..135fda36a2 100644 --- a/tests/functional/aws-node-sdk/test/object/getPartSize.js +++ b/tests/functional/aws-node-sdk/test/object/getPartSize.js @@ -30,8 +30,7 @@ const invalidPartNumbers = [-1, 0, maximumAllowedPartCount + 1]; let ETags = []; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function generateContent(partNumber) { @@ -45,25 +44,31 @@ describe('Part size tests with object head', () => { let uploadId; function headObject(fields, cb) { - s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: object, - ...fields, - })).then(data => { - cb(null, data); - }).catch(err => { - cb(err); - }); + s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: object, + ...fields, + }), + ) + .then(data => { + cb(null, data); + }) + .catch(err => { + cb(err); + }); } before(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const uploadResult = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: object - })); + const uploadResult = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: object, + }), + ); uploadId = uploadResult.UploadId; const uploadPromises = partNumbers.map(async partNumber => { const uploadPartParams = { @@ -77,16 +82,20 @@ describe('Part size tests with object head', () => { return result.ETag; }); ETags = await Promise.all(uploadPromises); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: emptyObject, - Body: '', - })); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: nonMpuObject, - Body: generateContent(0), - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: emptyObject, + Body: '', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: nonMpuObject, + Body: generateContent(0), + }), + ); const completeParams = { Bucket: bucket, Key: object, @@ -102,37 +111,42 @@ describe('Part size tests with object head', () => { }); after(async () => { - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: object - })); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: emptyObject - })); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: nonMpuObject - })); - await s3.send(new DeleteBucketCommand({ - Bucket: bucket - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: object, + }), + ); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: emptyObject, + }), + ); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: nonMpuObject, + }), + ); + await s3.send( + new DeleteBucketCommand({ + Bucket: bucket, + }), + ); }); - it('should return the total size of the object ' + - 'when --part-number is not used', done => { - const totalSize = partNumbers.reduce((total, current) => - total + (bodySize + current + 1), 0); - headObject({}, (err, data) => { - checkNoError(err); - assert.equal(totalSize, data.ContentLength); - done(); - }); + it('should return the total size of the object ' + 'when --part-number is not used', done => { + const totalSize = partNumbers.reduce((total, current) => total + (bodySize + current + 1), 0); + headObject({}, (err, data) => { + checkNoError(err); + assert.equal(totalSize, data.ContentLength); + done(); }); + }); partNumbers.forEach(part => { - it(`should return the size of part ${part + 1} ` + - `when --part-number is set to ${part + 1}`, done => { + it(`should return the size of part ${part + 1} ` + `when --part-number is set to ${part + 1}`, done => { const partNumber = Number.parseInt(part, 10) + 1; const partSize = bodySize + partNumber; headObject({ PartNumber: partNumber }, (err, data) => { @@ -144,8 +158,7 @@ describe('Part size tests with object head', () => { }); invalidPartNumbers.forEach(part => { - it(`should return an error when --part-number is set to ${part}`, - done => { + it(`should return an error when --part-number is set to ${part}`, done => { headObject({ PartNumber: part }, err => { assert.equal(err.$metadata.httpStatusCode, 400); done(); @@ -154,8 +167,7 @@ describe('Part size tests with object head', () => { }); it('should return an error when incorrect --part-number is used', done => { - headObject({ PartNumber: partNumbers.length + 1 }, - err => { + headObject({ PartNumber: partNumbers.length + 1 }, err => { checkError(err, '', 416); done(); }); @@ -174,7 +186,7 @@ describe('Part size tests with object head', () => { checkError(err, '', 416); done(); }); - }); + }); it('should return content-length requesting part 1 of non-MPU object', done => { headObject({ Key: nonMpuObject, PartNumber: 1 }, (err, data) => { diff --git a/tests/functional/aws-node-sdk/test/object/getRange.js b/tests/functional/aws-node-sdk/test/object/getRange.js index 236d0e7618..44efc3ee7f 100644 --- a/tests/functional/aws-node-sdk/test/object/getRange.js +++ b/tests/functional/aws-node-sdk/test/object/getRange.js @@ -1,9 +1,5 @@ const assert = require('assert'); -const { - GetObjectCommand, - CreateBucketCommand, - PutObjectCommand -} = require('@aws-sdk/client-s3'); +const { GetObjectCommand, CreateBucketCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); @@ -44,12 +40,13 @@ describe('aws-node-sdk range test of large end position', () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objName, - Body: Buffer.allocUnsafe(2890).fill(0, 0, 2800) - .fill(1, 2800), - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objName, + Body: Buffer.allocUnsafe(2890).fill(0, 0, 2800).fill(1, 2800), + }), + ); }); afterEach(async () => { @@ -59,15 +56,13 @@ describe('aws-node-sdk range test of large end position', () => { await bucketUtil.deleteOne(bucketName); }); - it('should get the final 90 bytes of a 2890 byte object for a byte ' + - 'range of 2800-', - done => endRangeTest('bytes=2800-', 'bytes 2800-2889/2890', done) + it('should get the final 90 bytes of a 2890 byte object for a byte ' + 'range of 2800-', done => + endRangeTest('bytes=2800-', 'bytes 2800-2889/2890', done), ); - it('should get the final 90 bytes of a 2890 byte object for a byte ' + - 'range of 2800-Number.MAX_SAFE_INTEGER', - done => endRangeTest(`bytes=2800-${Number.MAX_SAFE_INTEGER}`, - 'bytes 2800-2889/2890', done) + it( + 'should get the final 90 bytes of a 2890 byte object for a byte ' + 'range of 2800-Number.MAX_SAFE_INTEGER', + done => endRangeTest(`bytes=2800-${Number.MAX_SAFE_INTEGER}`, 'bytes 2800-2889/2890', done), ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/getRetention.js b/tests/functional/aws-node-sdk/test/object/getRetention.js index 3c97a0fbaa..682f409246 100644 --- a/tests/functional/aws-node-sdk/test/object/getRetention.js +++ b/tests/functional/aws-node-sdk/test/object/getRetention.js @@ -32,7 +32,6 @@ const expectedConfig = { RetainUntilDate: new Date(retainDate), }; - describe('GET object retention', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -43,23 +42,27 @@ describe('GET object retention', () => { beforeEach(async () => { process.stdout.write('Putting buckets and objects\n'); - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: objectName })); await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: noRetentionObject })); - + const res = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); versionId = res.VersionId; - + process.stdout.write('Putting object retention\n'); - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); }); afterEach(async () => { @@ -69,13 +72,14 @@ describe('GET object retention', () => { await bucketUtil.deleteMany([bucketName, unlockedBucket]); }); - it('should return AccessDenied putting retention with another account', - async () => { + it('should return AccessDenied putting retention with another account', async () => { try { - await otherAccountS3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - })); + await otherAccountS3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { checkError(err, 'AccessDenied', 403); @@ -84,10 +88,12 @@ describe('GET object retention', () => { it('should return NoSuchKey error if key does not exist', async () => { try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: 'thiskeydoesnotexist', - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: 'thiskeydoesnotexist', + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { checkError(err, 'NoSuchKey', 404); @@ -96,52 +102,60 @@ describe('GET object retention', () => { it('should return NoSuchVersion error if version does not exist', async () => { try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: '012345678901234567890123456789012', - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: '012345678901234567890123456789012', + }), + ); throw new Error('Expected NoSuchVersion error'); } catch (err) { checkError(err, 'NoSuchVersion', 404); } }); - it('should return MethodNotAllowed if object version is delete marker', - async () => { + it('should return MethodNotAllowed if object version is delete marker', async () => { const res = await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: res.VersionId, - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: res.VersionId, + }), + ); throw new Error('Expected MethodNotAllowed error'); } catch (err) { checkError(err, 'MethodNotAllowed', 405); } }); - it('should return InvalidRequest error getting retention to object ' + - 'in bucket with no object lock enabled', async () => { + it( + 'should return InvalidRequest error getting retention to object ' + 'in bucket with no object lock enabled', + async () => { + try { + await s3.send( + new GetObjectRetentionCommand({ + Bucket: unlockedBucket, + Key: objectName, + }), + ); + throw new Error('Expected InvalidRequest error'); + } catch (err) { + checkError(err, 'InvalidRequest', 400); + } + }, + ); + + it('should return NoSuchObjectLockConfiguration if no retention set', async () => { try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: unlockedBucket, - Key: objectName, - })); - throw new Error('Expected InvalidRequest error'); - } catch (err) { - checkError(err, 'InvalidRequest', 400); - } - }); - - it('should return NoSuchObjectLockConfiguration if no retention set', - async () => { - try { - await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: noRetentionObject, - })); + await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: noRetentionObject, + }), + ); throw new Error('Expected NoSuchObjectLockConfiguration error'); } catch (err) { checkError(err, 'NoSuchObjectLockConfiguration', 404); @@ -149,13 +163,14 @@ describe('GET object retention', () => { }); it('should get object retention', async () => { - const res = await s3.send(new GetObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - })); + const res = await s3.send( + new GetObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(res.Retention, expectedConfig); - await changeLockPromise([ - { bucket: bucketName, key: objectName, versionId }], ''); + await changeLockPromise([{ bucket: bucketName, key: objectName, versionId }], ''); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/initiateMPU.js b/tests/functional/aws-node-sdk/test/object/initiateMPU.js index eb6416973a..97fb816895 100644 --- a/tests/functional/aws-node-sdk/test/object/initiateMPU.js +++ b/tests/functional/aws-node-sdk/test/object/initiateMPU.js @@ -8,8 +8,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const genMaxSizeMetaHeaders - = require('../../lib/utility/genMaxSizeMetaHeaders'); +const genMaxSizeMetaHeaders = require('../../lib/utility/genMaxSizeMetaHeaders'); const { generateMultipleTagQuery } = require('../../lib/utility/tagging'); const bucket = `initiatempubucket${Date.now()}`; @@ -28,59 +27,69 @@ describe('Initiate MPU', () => { afterEach(async () => await bucketUtil.deleteOne(bucket)); - it('should return InvalidRedirectLocation if initiate MPU ' + - 'with x-amz-website-redirect-location header that does not start ' + - 'with \'http://\', \'https://\' or \'/\'', async () => { - const params = { - Bucket: bucket, - Key: key, - WebsiteRedirectLocation: 'google.com' - }; - - try { - await s3.send(new CreateMultipartUploadCommand(params)); - throw new Error('Expected InvalidRedirectLocation error'); - } catch (err) { - assert.strictEqual(err.name, 'InvalidRedirectLocation'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } - }); + it( + 'should return InvalidRedirectLocation if initiate MPU ' + + 'with x-amz-website-redirect-location header that does not start ' + + "with 'http://', 'https://' or '/'", + async () => { + const params = { + Bucket: bucket, + Key: key, + WebsiteRedirectLocation: 'google.com', + }; - it('should return InvalidStorageClass error when x-amz-storage-class header is provided ' + - 'and not equal to STANDARD', done => { - s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - StorageClass: 'COLD', - })).then(() => { - throw new Error('Expected InvalidStorageClass error'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidStorageClass'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + try { + await s3.send(new CreateMultipartUploadCommand(params)); + throw new Error('Expected InvalidRedirectLocation error'); + } catch (err) { + assert.strictEqual(err.name, 'InvalidRedirectLocation'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } + }, + ); + + it( + 'should return InvalidStorageClass error when x-amz-storage-class header is provided ' + + 'and not equal to STANDARD', + done => { + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + StorageClass: 'COLD', + }), + ) + .then(() => { + throw new Error('Expected InvalidStorageClass error'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidStorageClass'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); it('should return KeyTooLong error when key is longer than 915 bytes', done => { - s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: 'a'.repeat(916) })) - .catch(err => { + s3.send(new CreateMultipartUploadCommand({ Bucket: bucket, Key: 'a'.repeat(916) })).catch(err => { assert.strictEqual(err.name, 'KeyTooLong'); assert.strictEqual(err.$metadata.httpStatusCode, 400); done(); }); }); - it('should return error if initiating MPU w/ > 2KB user-defined md', - async () => { + it('should return error if initiating MPU w/ > 2KB user-defined md', async () => { const metadata = genMaxSizeMetaHeaders(); const params = { Bucket: bucket, Key: key, Metadata: metadata }; const data = await s3.send(new CreateMultipartUploadCommand(params)); const uploadId = data.UploadId; - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); metadata.header0 = `${metadata.header0}${'0'}`; try { await s3.send(new CreateMultipartUploadCommand(params)); @@ -91,57 +100,62 @@ describe('Initiate MPU', () => { } }); - it('should return error if initiating MPU w/ > 2KB user-defined md', - async () => { - const metadata = genMaxSizeMetaHeaders(); - const params = { Bucket: bucket, Key: key, Metadata: metadata }; - const data = await s3.send(new CreateMultipartUploadCommand(params)); - const uploadId = data.UploadId; - await s3.send(new AbortMultipartUploadCommand({ + it('should return error if initiating MPU w/ > 2KB user-defined md', async () => { + const metadata = genMaxSizeMetaHeaders(); + const params = { Bucket: bucket, Key: key, Metadata: metadata }; + const data = await s3.send(new CreateMultipartUploadCommand(params)); + const uploadId = data.UploadId; + await s3.send( + new AbortMultipartUploadCommand({ Bucket: bucket, Key: key, UploadId: uploadId, - })); - metadata.header0 = `${metadata.header0}${'0'}`; - try { - await s3.send(new CreateMultipartUploadCommand(params)); - throw new Error('Expected MetadataTooLarge error'); - } catch (err) { - assert.strictEqual(err.name, 'MetadataTooLarge'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - } + }), + ); + metadata.header0 = `${metadata.header0}${'0'}`; + try { + await s3.send(new CreateMultipartUploadCommand(params)); + throw new Error('Expected MetadataTooLarge error'); + } catch (err) { + assert.strictEqual(err.name, 'MetadataTooLarge'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + } }); describe('with tag set', () => { - it('should be able to put object with 10 tags', - async () => { + it('should be able to put object with 10 tags', async () => { const taggingConfig = generateMultipleTagQuery(10); - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: taggingConfig, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: taggingConfig, + }), + ); }); it('should allow putting 50 tags', async () => { const taggingConfig = generateMultipleTagQuery(50); - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: taggingConfig, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: taggingConfig, + }), + ); }); - it('should return BadRequest if putting more that 50 tags', - async () => { + it('should return BadRequest if putting more that 50 tags', async () => { const taggingConfig = generateMultipleTagQuery(51); - + try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: taggingConfig, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: taggingConfig, + }), + ); throw new Error('Expected BadRequest error'); } catch (err) { assert.strictEqual(err.name, 'BadRequest'); @@ -149,16 +163,17 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument creating mpu tag with ' + - 'invalid characters: %', async () => { + it('should return InvalidArgument creating mpu tag with ' + 'invalid characters: %', async () => { const value = 'value1%'; - + try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: `key1=${value}`, - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: `key1=${value}`, + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -166,14 +181,15 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument creating mpu with ' + - 'bad encoded tags', async () => { + it('should return InvalidArgument creating mpu with ' + 'bad encoded tags', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: 'key1==value1', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: 'key1==value1', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -183,11 +199,13 @@ describe('Initiate MPU', () => { it('should return InvalidArgument if tag with no key', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: '=value1', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: '=value1', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -195,14 +213,15 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument if using the same key twice', - async () => { + it('should return InvalidArgument if using the same key twice', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - Tagging: 'key1=value1&key1=value2', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + Tagging: 'key1=value1&key1=value2', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); @@ -210,14 +229,15 @@ describe('Initiate MPU', () => { } }); - it('should return InvalidArgument if using the same key twice ' + - 'and empty tags', async () => { + it('should return InvalidArgument if using the same key twice ' + 'and empty tags', async () => { try { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2', - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2', + }), + ); throw new Error('Expected InvalidArgument error'); } catch (err) { assert.strictEqual(err.name, 'InvalidArgument'); diff --git a/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js b/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js index 628e810792..620d7b9d26 100644 --- a/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/listMpuChecksum.js @@ -64,16 +64,24 @@ describe('ListMultipartUploads checksum fields', () => let uploadId; before(async () => { - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: algo, - })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + }), + ); uploadId = res.UploadId; }); after(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); }); it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${expectedType}`, async () => { @@ -91,9 +99,14 @@ describe('ListMultipartUploads checksum fields', () => let uploadId; before(async () => { - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC32', ChecksumType: 'FULL_OBJECT', - })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'CRC32', + ChecksumType: 'FULL_OBJECT', + }), + ); uploadId = res.UploadId; }); @@ -114,21 +127,30 @@ describe('ListMultipartUploads checksum fields', () => const uploads = []; before(async () => { - const noChecksum = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: `${key}-no-checksum`, - })); + const noChecksum = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: `${key}-no-checksum`, + }), + ); uploads.push({ key: `${key}-no-checksum`, uploadId: noChecksum.UploadId }); - const withChecksum = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: `${key}-with-checksum`, ChecksumAlgorithm: 'SHA256', - })); + const withChecksum = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: `${key}-with-checksum`, + ChecksumAlgorithm: 'SHA256', + }), + ); uploads.push({ key: `${key}-with-checksum`, uploadId: withChecksum.UploadId }); }); after(async () => { - await Promise.all(uploads.map(u => - s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.key, UploadId: u.uploadId })) - )); + await Promise.all( + uploads.map(u => + s3.send(new AbortMultipartUploadCommand({ Bucket: bucket, Key: u.key, UploadId: u.uploadId })), + ), + ); }); it('should return checksum fields only for uploads that have them', async () => { @@ -144,5 +166,4 @@ describe('ListMultipartUploads checksum fields', () => assert.strictEqual(withChecksumUpload.ChecksumType, 'COMPOSITE'); }); }); - }) -); + })); diff --git a/tests/functional/aws-node-sdk/test/object/listParts.js b/tests/functional/aws-node-sdk/test/object/listParts.js index f670b33eb8..d02cb93fb2 100644 --- a/tests/functional/aws-node-sdk/test/object/listParts.js +++ b/tests/functional/aws-node-sdk/test/object/listParts.js @@ -24,35 +24,59 @@ describe('List parts', () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = res.UploadId; - await s3.send(new UploadPartCommand({ Bucket: bucket, Key: key, - PartNumber: 1, UploadId: uploadId, Body: bodyFirstPart, - })); - const secondRes = await s3.send(new UploadPartCommand({ - Bucket: bucket, Key: key, - PartNumber: 2, UploadId: uploadId, Body: bodySecondPart, - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: bodyFirstPart, + }), + ); + const secondRes = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 2, + UploadId: uploadId, + Body: bodySecondPart, + }), + ); secondEtag = secondRes.ETag; }); afterEach(async () => { process.stdout.write('Emptying bucket'); - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); await bucketUtil.empty(bucket); process.stdout.write('Deleting bucket'); await bucketUtil.deleteOne(bucket); }); - it('should only list the second part', () => s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: key, - PartNumberMarker: '1', - UploadId: uploadId, - })).then(data => { + it('should only list the second part', () => + s3 + .send( + new ListPartsCommand({ + Bucket: bucket, + Key: key, + PartNumberMarker: '1', + UploadId: uploadId, + }), + ) + .then(data => { assert.strictEqual(data.Parts[0].PartNumber, 2); assert.strictEqual(data.Parts[0].Size, 20); assert.strictEqual(`${data.Parts[0].ETag}`, secondEtag); @@ -64,36 +88,59 @@ describe('List parts', () => { function createPart(sigCfg, bucketUtil, s3, key) { let uploadId; - return s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key }))) - .then(res => { - uploadId = res.UploadId; - return s3.send(new UploadPartCommand({ Bucket: bucket, Key: key, - PartNumber: 1, UploadId: uploadId, Body: bodyFirstPart })); - }) - .then(() => Promise.resolve(uploadId)); + return s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ), + ) + .then(res => { + uploadId = res.UploadId; + return s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: 1, + UploadId: uploadId, + Body: bodyFirstPart, + }), + ); + }) + .then(() => Promise.resolve(uploadId)); } function deletePart(s3, bucketUtil, key, uploadId) { process.stdout.write('Emptying bucket'); - return s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: uploadId, - })) - .then(() => bucketUtil.empty(bucket)) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }); + return s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) + .then(() => bucketUtil.empty(bucket)) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }); } function testFunc(s3, bucket, key, uploadId) { - return s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })).then(data => { + return s3 + .send( + new ListPartsCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ) + .then(data => { assert.strictEqual(data.Key, key); }); } @@ -106,17 +153,15 @@ describe('List parts - object keys with special characters: `&`', () => { const key = '&'; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `&` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `&` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); @@ -128,39 +173,35 @@ describe('List parts - object keys with special characters: `"`', () => { const key = '"quot'; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `"` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `"` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); -describe('List parts - object keys with special characters: `\'`', () => { +describe("List parts - object keys with special characters: `'`", () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; let uploadId; - const key = '\'apos'; + const key = "'apos"; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `\'` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it("should list parts of an object with `'` in its key", () => testFunc(s3, bucket, key, uploadId)); }); }); @@ -172,17 +213,15 @@ describe('List parts - object keys with special characters: `<`', () => { const key = ' - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `<` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `<` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); @@ -194,16 +233,14 @@ describe('List parts - object keys with special characters: `>`', () => { const key = '>gt'; beforeEach(() => - createPart(sigCfg, bucketUtil, s3, key) - .then(res => { + createPart(sigCfg, bucketUtil, s3, key).then(res => { uploadId = res; return Promise.resolve(); - }) + }), ); afterEach(() => deletePart(s3, bucketUtil, key, uploadId)); - it('should list parts of an object with `>` in its key', - () => testFunc(s3, bucket, key, uploadId)); + it('should list parts of an object with `>` in its key', () => testFunc(s3, bucket, key, uploadId)); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/listPartsChecksum.js b/tests/functional/aws-node-sdk/test/object/listPartsChecksum.js index 4c8d5a33dd..48884515fc 100644 --- a/tests/functional/aws-node-sdk/test/object/listPartsChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/listPartsChecksum.js @@ -14,10 +14,7 @@ const { algorithms } = require('../../../../../lib/api/apiUtils/integrity/valida const bucket = `list-parts-checksum-test-${Date.now()}`; const defaultKey = 'default-no-checksum'; -const checksumBodies = [ - Buffer.from('first checksummed part', 'utf8'), - Buffer.from('second checksummed part', 'utf8'), -]; +const checksumBodies = [Buffer.from('first checksummed part', 'utf8'), Buffer.from('second checksummed part', 'utf8')]; const defaultBodies = [ Buffer.from('first default checksum part', 'utf8'), Buffer.from('second default checksum part', 'utf8'), @@ -56,13 +53,14 @@ describe('ListParts checksum fields', () => const openMPUs = []; async function abortUpload(key, uploadId) { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); - const index = openMPUs.findIndex(upload => - upload.key === key && upload.uploadId === uploadId); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); + const index = openMPUs.findIndex(upload => upload.key === key && upload.uploadId === uploadId); if (index !== -1) { openMPUs.splice(index, 1); } @@ -78,97 +76,116 @@ describe('ListParts checksum fields', () => }); after(async () => { - await Promise.all(openMPUs.map(upload => - s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: upload.key, - UploadId: upload.uploadId, - })).catch(() => undefined))); + await Promise.all( + openMPUs.map(upload => + s3 + .send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: upload.key, + UploadId: upload.uploadId, + }), + ) + .catch(() => undefined), + ), + ); await bucketUtil.empty(bucket); await s3.send(new DeleteBucketCommand({ Bucket: bucket })); }); - for (const [checksumAlgorithm, checksumTypes] of - Object.entries(checksumTypesByAlgorithm)) { + for (const [checksumAlgorithm, checksumTypes] of Object.entries(checksumTypesByAlgorithm)) { for (const checksumType of checksumTypes) { - it(`should include ${checksumAlgorithm}/${checksumType} root ` + - 'and part checksum fields', async () => { - const key = `explicit-${checksumAlgorithm}-${checksumType}`; - const checksumField = - checksumFieldByAlgorithm[checksumAlgorithm]; - const internalAlgorithm = checksumAlgorithm.toLowerCase(); - const { UploadId } = - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - ChecksumAlgorithm: checksumAlgorithm, - ChecksumType: checksumType, - })); - openMPUs.push({ key, uploadId: UploadId }); - - const partChecksums = await Promise.all( - checksumBodies.map(body => - algorithms[internalAlgorithm].digest(body))); - - await Promise.all(checksumBodies.map((body, index) => - s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - UploadId, - PartNumber: index + 1, - Body: body, - [checksumField]: partChecksums[index], - })))); - - const partList = await s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: key, - UploadId, - })); - - assert.strictEqual(partList.ChecksumAlgorithm, - checksumAlgorithm); - assert.strictEqual(partList.ChecksumType, checksumType); - assert.strictEqual(partList.Parts.length, - checksumBodies.length); - partList.Parts.forEach((part, index) => { - assert.strictEqual(part.PartNumber, index + 1); - assert.strictEqual(part[checksumField], - partChecksums[index]); - }); - - await abortUpload(key, UploadId); - }); + it( + `should include ${checksumAlgorithm}/${checksumType} root ` + 'and part checksum fields', + async () => { + const key = `explicit-${checksumAlgorithm}-${checksumType}`; + const checksumField = checksumFieldByAlgorithm[checksumAlgorithm]; + const internalAlgorithm = checksumAlgorithm.toLowerCase(); + const { UploadId } = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: checksumAlgorithm, + ChecksumType: checksumType, + }), + ); + openMPUs.push({ key, uploadId: UploadId }); + + const partChecksums = await Promise.all( + checksumBodies.map(body => algorithms[internalAlgorithm].digest(body)), + ); + + await Promise.all( + checksumBodies.map((body, index) => + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId, + PartNumber: index + 1, + Body: body, + [checksumField]: partChecksums[index], + }), + ), + ), + ); + + const partList = await s3.send( + new ListPartsCommand({ + Bucket: bucket, + Key: key, + UploadId, + }), + ); + + assert.strictEqual(partList.ChecksumAlgorithm, checksumAlgorithm); + assert.strictEqual(partList.ChecksumType, checksumType); + assert.strictEqual(partList.Parts.length, checksumBodies.length); + partList.Parts.forEach((part, index) => { + assert.strictEqual(part.PartNumber, index + 1); + assert.strictEqual(part[checksumField], partChecksums[index]); + }); + + await abortUpload(key, UploadId); + }, + ); } } - it('should omit default checksum fields when no checksum headers are sent', - async () => { - const { UploadId } = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: defaultKey, - })); + it('should omit default checksum fields when no checksum headers are sent', async () => { + const { UploadId } = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: defaultKey, + }), + ); openMPUs.push({ key: defaultKey, uploadId: UploadId }); - await Promise.all(defaultBodies.map((body, index) => - s3.send(new UploadPartCommand({ + await Promise.all( + defaultBodies.map((body, index) => + s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: defaultKey, + UploadId, + PartNumber: index + 1, + Body: body, + }), + ), + ), + ); + + const partList = await s3.send( + new ListPartsCommand({ Bucket: bucket, Key: defaultKey, UploadId, - PartNumber: index + 1, - Body: body, - })))); - - const partList = await s3.send(new ListPartsCommand({ - Bucket: bucket, - Key: defaultKey, - UploadId, - })); + }), + ); assert.strictEqual(partList.Parts.length, defaultBodies.length); assertNoListPartsChecksum(partList); await abortUpload(defaultKey, UploadId); }); - }) -); + })); diff --git a/tests/functional/aws-node-sdk/test/object/mpu.js b/tests/functional/aws-node-sdk/test/object/mpu.js index 58eaebd8d3..4bf8f1b8a2 100644 --- a/tests/functional/aws-node-sdk/test/object/mpu.js +++ b/tests/functional/aws-node-sdk/test/object/mpu.js @@ -1,4 +1,3 @@ - const assert = require('assert'); const { CreateBucketCommand, @@ -42,22 +41,22 @@ function getExpectedObj(res, data) { NextUploadIdMarker: uploadId, MaxUploads: maxUploads, IsTruncated: false, - Uploads: [{ - UploadId: uploadId, - Key: objectKey, - Initiated: initiated, - StorageClass: 'STANDARD', - Owner: - { - DisplayName: displayName, - ID: userId, - }, - Initiator: + Uploads: [ { - DisplayName: displayName, - ID: userId, + UploadId: uploadId, + Key: objectKey, + Initiated: initiated, + StorageClass: 'STANDARD', + Owner: { + DisplayName: displayName, + ID: userId, + }, + Initiator: { + DisplayName: displayName, + ID: userId, + }, }, - }], + ], }; // If no `prefixVal` is given, it should not be included in the response. @@ -93,20 +92,24 @@ describe('aws-node-sdk test suite of listMultipartUploads', () => // The owner of the bucket will also be the MPU upload owner. data.displayName = ownerRes.DisplayName; data.userId = ownerRes.ID; - - const mpuRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - })); - data.uploadId = mpuRes.UploadId; + + const mpuRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + }), + ); + data.uploadId = mpuRes.UploadId; }); afterEach(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: objectKey, - UploadId: data.uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: objectKey, + UploadId: data.uploadId, + }), + ); await bucketUtil.empty(bucket); await bucketUtil.deleteOne(bucket); }); @@ -122,23 +125,26 @@ describe('aws-node-sdk test suite of listMultipartUploads', () => data.delimiter = 'test-delimiter'; data.maxUploads = 1; // eslint-disable-next-line no-unused-vars - const {$metadata, ...res } = await s3.send(new ListMultipartUploadsCommand({ - Bucket: bucket, - Prefix: 'to', - Delimiter: 'test-delimiter', - MaxUploads: 1, - })); + const { $metadata, ...res } = await s3.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + Prefix: 'to', + Delimiter: 'test-delimiter', + MaxUploads: 1, + }), + ); checkValues(res, data); }); it('should list 0 multipart uploads when MaxUploads is 0', async () => { data.maxUploads = 0; // eslint-disable-next-line no-unused-vars - const { $metadata , ...res } = await s3.send(new ListMultipartUploadsCommand({ - Bucket: bucket, - MaxUploads: 0, - })); + const { $metadata, ...res } = await s3.send( + new ListMultipartUploadsCommand({ + Bucket: bucket, + MaxUploads: 0, + }), + ); checkValues(res, data); }); - }) -); + })); diff --git a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js index d68d4cc973..4d3bef75d7 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuChecksum.js +++ b/tests/functional/aws-node-sdk/test/object/mpuChecksum.js @@ -37,9 +37,13 @@ describe('CreateMultipartUpload checksum headers', () => }); after(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: res.UploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: res.UploadId, + }), + ); }); it('should not return ChecksumAlgorithm and ChecksumType', () => { @@ -70,15 +74,23 @@ describe('CreateMultipartUpload checksum headers', () => let res; before(async () => { - res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: algo, - })); + res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + }), + ); }); after(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: res.UploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: res.UploadId, + }), + ); }); it(`should return ChecksumAlgorithm ${algo} and default ChecksumType to ${expectedType}`, () => { @@ -113,15 +125,24 @@ describe('CreateMultipartUpload checksum headers', () => let res; before(async () => { - res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: algo, ChecksumType: type, - })); + res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: algo, + ChecksumType: type, + }), + ); }); after(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, Key: key, UploadId: res.UploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: res.UploadId, + }), + ); }); it(`should return ChecksumAlgorithm ${algo} and ChecksumType ${type}`, () => { @@ -143,9 +164,14 @@ describe('CreateMultipartUpload checksum headers', () => describe('error cases', () => { it('should reject FULL_OBJECT with SHA256', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA256', ChecksumType: 'FULL_OBJECT', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'SHA256', + ChecksumType: 'FULL_OBJECT', + }), + ); assert.fail('Expected error'); } catch (err) { assert.strictEqual(err.name, 'InvalidRequest'); @@ -154,9 +180,14 @@ describe('CreateMultipartUpload checksum headers', () => it('should reject FULL_OBJECT with SHA1', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: 'SHA1', ChecksumType: 'FULL_OBJECT', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'SHA1', + ChecksumType: 'FULL_OBJECT', + }), + ); assert.fail('Expected error'); } catch (err) { assert.strictEqual(err.name, 'InvalidRequest'); @@ -165,14 +196,18 @@ describe('CreateMultipartUpload checksum headers', () => it('should reject COMPOSITE with CRC64NVME', async () => { try { - await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, Key: key, ChecksumAlgorithm: 'CRC64NVME', ChecksumType: 'COMPOSITE', - })); + await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + ChecksumAlgorithm: 'CRC64NVME', + ChecksumType: 'COMPOSITE', + }), + ); assert.fail('Expected error'); } catch (err) { assert.strictEqual(err.name, 'InvalidRequest'); } }); }); - }) -); + })); diff --git a/tests/functional/aws-node-sdk/test/object/mpuOrder.js b/tests/functional/aws-node-sdk/test/object/mpuOrder.js index 7ef4ef3d66..fea9648b74 100644 --- a/tests/functional/aws-node-sdk/test/object/mpuOrder.js +++ b/tests/functional/aws-node-sdk/test/object/mpuOrder.js @@ -23,13 +23,13 @@ function checkError(err, statusCode, code) { const body = Buffer.alloc(1024 * 1024 * 5, 'a'); const testsOrder = [ - { values: [3, 8, 1000], err: false }, - { values: [8, 3, 1000], err: true }, - { values: [8, 1000, 3], err: true }, - { values: [1000, 3, 8], err: true }, - { values: [3, 1000, 8], err: true }, - { values: [1000, 8, 3], err: true }, - { values: [3, 3, 1000], err: true }, + { values: [3, 8, 1000], err: false }, + { values: [8, 3, 1000], err: true }, + { values: [8, 1000, 3], err: true }, + { values: [1000, 3, 8], err: true }, + { values: [3, 1000, 8], err: true }, + { values: [1000, 8, 3], err: true }, + { values: [3, 3, 1000], err: true }, ]; describe('More MPU tests', () => { @@ -41,33 +41,41 @@ describe('More MPU tests', () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const mpuRes = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: object - })); + const mpuRes = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: object, + }), + ); this.currentTest.UploadId = mpuRes.UploadId; - const part1000Res = await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: object, - PartNumber: 1000, - Body: body, - UploadId: this.currentTest.UploadId - })); + const part1000Res = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: object, + PartNumber: 1000, + Body: body, + UploadId: this.currentTest.UploadId, + }), + ); this.currentTest.Etag = part1000Res.ETag; - await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: object, - PartNumber: 3, - Body: body, - UploadId: this.currentTest.UploadId - })); - await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: object, - PartNumber: 8, - Body: body, - UploadId: this.currentTest.UploadId - })); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: object, + PartNumber: 3, + Body: body, + UploadId: this.currentTest.UploadId, + }), + ); + await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: object, + PartNumber: 8, + Body: body, + UploadId: this.currentTest.UploadId, + }), + ); }); afterEach(async () => { @@ -76,47 +84,53 @@ describe('More MPU tests', () => { }); testsOrder.forEach(testOrder => { - it('should complete MPU by concatenating the parts in ' + - `the following order: ${testOrder.values}`, async function itF() { - try { - await s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: object, - MultipartUpload: { - Parts: [ - { - ETag: this.test.Etag, - PartNumber: testOrder.values[0], + it( + 'should complete MPU by concatenating the parts in ' + `the following order: ${testOrder.values}`, + async function itF() { + try { + await s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: object, + MultipartUpload: { + Parts: [ + { + ETag: this.test.Etag, + PartNumber: testOrder.values[0], + }, + { + ETag: this.test.Etag, + PartNumber: testOrder.values[1], + }, + { + ETag: this.test.Etag, + PartNumber: testOrder.values[2], + }, + ], }, - { - ETag: this.test.Etag, - PartNumber: testOrder.values[1], - }, - { - ETag: this.test.Etag, - PartNumber: testOrder.values[2], - }, - ], - }, - UploadId: this.test.UploadId - })); - - if (testOrder.err) { - throw new Error('Expected InvalidPartOrder error but operation succeeded'); - } - } catch (err) { - if (testOrder.err) { - checkError(err, 400, 'InvalidPartOrder'); - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: object, - UploadId: this.test.UploadId, - })); - } else { - throw err; + UploadId: this.test.UploadId, + }), + ); + + if (testOrder.err) { + throw new Error('Expected InvalidPartOrder error but operation succeeded'); + } + } catch (err) { + if (testOrder.err) { + checkError(err, 400, 'InvalidPartOrder'); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: object, + UploadId: this.test.UploadId, + }), + ); + } else { + throw err; + } } - } - }); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js index 7aa6d9babf..4d591b0ff2 100644 --- a/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js +++ b/tests/functional/aws-node-sdk/test/object/multiObjectDelete.js @@ -23,8 +23,7 @@ const bucketName = 'multi-object-delete-234-634'; const key = 'key'; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function sortList(list) { @@ -32,7 +31,7 @@ function sortList(list) { // Handle both string arrays and object arrays const keyA = typeof a === 'string' ? a : a.Key; const keyB = typeof b === 'string' ? b : b.Key; - + // Extract numeric part from keys like 'key1', 'key2', 'key10', etc. const getNumber = key => parseInt(key.replace(/^key/, ''), 10); const numA = getNumber(keyA); @@ -43,7 +42,7 @@ function sortList(list) { function createObjectsList(size, versionIds) { const objects = []; - for (let i = 1; i < (size + 1); i++) { + for (let i = 1; i < size + 1; i++) { objects.push({ Key: `${key}${i}`, }); @@ -80,11 +79,13 @@ describe('Multi-Object Delete Success', function success() { await Promise.race(queued); queued.splice(0, queued.findIndex(p => p === queued[0]) + 1); } - const result = s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: key, - Body: 'somebody', - })); + const result = s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: 'somebody', + }), + ); queued.push(result); return result; }; @@ -103,39 +104,48 @@ describe('Multi-Object Delete Success', function success() { it('should batch delete 1000 objects', done => { const objects = createObjectsList(1000); - s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - if (this.httpResponse?.body?.toString() - .indexOf(' obj.Key)), sortList(objects.map(obj => obj.Key))); - return done(); - }).catch(err => done(err)); + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { + if (this.httpResponse?.body?.toString().indexOf(' obj.Key)), + sortList(objects.map(obj => obj.Key)), + ); + return done(); + }) + .catch(err => done(err)); }); it('should batch delete 1000 objects quietly', done => { const objects = createObjectsList(1000); - s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true, - }, - })).then(res => { - if (this.httpResponse?.body?.toString() - .indexOf(' done(err)); + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true, + }, + }), + ) + .then(res => { + if (this.httpResponse?.body?.toString().indexOf(' done(err)); }); }); @@ -147,8 +157,7 @@ describe('Multi-Object Delete Error Responses', () => { beforeEach(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .catch(err => { + return s3.send(new CreateBucketCommand({ Bucket: bucketName })).catch(err => { process.stdout.write(`Error creating bucket: ${err}\n`); throw err; }); @@ -159,57 +168,71 @@ describe('Multi-Object Delete Error Responses', () => { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return error if request deletion of more than 1000 objects', - () => { - const objects = createObjectsList(1001); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })).catch(err => { + it('should return error if request deletion of more than 1000 objects', () => { + const objects = createObjectsList(1001); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ) + .catch(err => { checkError(err, 'MalformedXML', 400); }); - }); + }); - it('should return error if request deletion of 0 objects', - () => { - const objects = createObjectsList(0); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })).catch(err => { + it('should return error if request deletion of 0 objects', () => { + const objects = createObjectsList(0); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ) + .catch(err => { checkError(err, 'MalformedXML', 400); }); - }); + }); - it('should return no error if try to delete non-existent objects', - () => { - const objects = createObjectsList(1000); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })).then(res => { + it('should return no error if try to delete non-existent objects', () => { + const objects = createObjectsList(1000); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ) + .then(res => { assert.strictEqual(res.Deleted.length, 1000); - }).catch(err => { + }) + .catch(err => { checkNoError(err); }); - }); + }); it('should return error if no such bucket', () => { const objects = createObjectsList(1); - return s3.send(new DeleteObjectsCommand({ - Bucket: 'nosuchbucket2323292093', - Delete: { - Objects: objects, - }, - })).catch(err => { - checkError(err, 'NoSuchBucket', 404); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: 'nosuchbucket2323292093', + Delete: { + Objects: objects, + }, + }), + ) + .catch(err => { + checkError(err, 'NoSuchBucket', 404); + }); }); }); }); @@ -224,26 +247,30 @@ describe('Multi-Object Delete Access', function access() { signatureVersion: 'v4', }); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }) - .then(() => { - const createObjects = []; - for (let i = 1; i < 501; i++) { - createObjects.push(s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: `${key}${i}`, - Body: 'somebody', - }))); - } - return Promise.all(createObjects) + return s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) .catch(err => { - process.stdout.write(`Error creating objects: ${err}\n`); + process.stdout.write(`Error creating bucket: ${err}\n`); throw err; + }) + .then(() => { + const createObjects = []; + for (let i = 1; i < 501; i++) { + createObjects.push( + s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: `${key}${i}`, + Body: 'somebody', + }), + ), + ); + } + return Promise.all(createObjects).catch(err => { + process.stdout.write(`Error creating objects: ${err}\n`); + throw err; + }); }); - }); }); after(async () => { @@ -251,8 +278,7 @@ describe('Multi-Object Delete Access', function access() { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return access denied error for each object where no acl ' + - 'permission', () => { + it('should return access denied error for each object where no acl ' + 'permission', () => { const objects = createObjectsList(500); const errorList = createObjectsList(500); errorList.forEach(obj => { @@ -260,38 +286,47 @@ describe('Multi-Object Delete Access', function access() { item.Code = 'AccessDenied'; item.Message = 'Access Denied'; }); - return otherAccountS3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - assert.strictEqual(res.Deleted, undefined); - assert.strictEqual(res.Errors.length, 500); - assert.deepStrictEqual(sortList(res.Errors), sortList(errorList)); - }).catch(err => { - checkNoError(err); - }); + return otherAccountS3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { + assert.strictEqual(res.Deleted, undefined); + assert.strictEqual(res.Errors.length, 500); + assert.deepStrictEqual(sortList(res.Errors), sortList(errorList)); + }) + .catch(err => { + checkNoError(err); + }); }); it('should batch delete objects where requester has permission', () => { const objects = createObjectsList(500); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - assert.strictEqual(res.Deleted.length, 500); - }).catch(err => { - checkNoError(err); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { + assert.strictEqual(res.Deleted.length, 500); + }) + .catch(err => { + checkNoError(err); + }); }); }); - describe('Multi-Object Delete with Object Lock', () => { let bucketUtil; let s3; @@ -303,45 +338,56 @@ describe('Multi-Object Delete with Object Lock', () => { signatureVersion: 'v4', }); s3 = bucketUtil.s3; - return s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })) - .then(() => s3.send(new PutObjectLockConfigurationCommand({ - Bucket: bucketName, - ObjectLockConfiguration: { - ObjectLockEnabled: 'Enabled', - Rule: { - DefaultRetention: { - Days: 1, - Mode: 'GOVERNANCE', - }, - }, - }, - }))) - .catch(err => { - process.stdout.write(`Error creating bucket: ${err}\n`); - throw err; - }) - .then(() => { - for (let i = 1; i < 6; i++) { - createObjects.push(s3.send(new PutObjectCommand({ + return s3 + .send( + new CreateBucketCommand({ Bucket: bucketName, - Key: `${key}${i}`, - Body: 'somebody', - }))); - } - return Promise.all(createObjects) - .then(res => { - res.forEach(r => { - versionIds.push(r.VersionId); - }); - }) + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => + s3.send( + new PutObjectLockConfigurationCommand({ + Bucket: bucketName, + ObjectLockConfiguration: { + ObjectLockEnabled: 'Enabled', + Rule: { + DefaultRetention: { + Days: 1, + Mode: 'GOVERNANCE', + }, + }, + }, + }), + ), + ) .catch(err => { - process.stdout.write(`Error creating objects: ${err}\n`); + process.stdout.write(`Error creating bucket: ${err}\n`); throw err; + }) + .then(() => { + for (let i = 1; i < 6; i++) { + createObjects.push( + s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: `${key}${i}`, + Body: 'somebody', + }), + ), + ); + } + return Promise.all(createObjects) + .then(res => { + res.forEach(r => { + versionIds.push(r.VersionId); + }); + }) + .catch(err => { + process.stdout.write(`Error creating objects: ${err}\n`); + throw err; + }); }); - }); }); after(async () => { @@ -351,45 +397,61 @@ describe('Multi-Object Delete with Object Lock', () => { it('should not delete locked objects', () => { const objects = createObjectsList(5, versionIds); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })).then(res => { - assert.strictEqual(res.Errors.length, 5); - res.Errors.forEach(err => assert.strictEqual(err.Code, 'AccessDenied')); - }); - }); - - it('should not delete locked objects with GOVERNANCE ' + - 'retention mode and bypass header when object is legal hold enabled', () => { - const objects = createObjectsList(5, versionIds); - const putObjectLegalHolds = []; - for (let i = 1; i < 6; i++) { - putObjectLegalHolds.push(s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucketName, - Key: `${key}${i}`, - LegalHold: { - Status: 'ON', - }, - }))); - } - return Promise.all(putObjectLegalHolds) - .then(() => s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - BypassGovernanceRetention: true, - }))).then(res => { + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ) + .then(res => { assert.strictEqual(res.Errors.length, 5); res.Errors.forEach(err => assert.strictEqual(err.Code, 'AccessDenied')); }); }); + it( + 'should not delete locked objects with GOVERNANCE ' + + 'retention mode and bypass header when object is legal hold enabled', + () => { + const objects = createObjectsList(5, versionIds); + const putObjectLegalHolds = []; + for (let i = 1; i < 6; i++) { + putObjectLegalHolds.push( + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucketName, + Key: `${key}${i}`, + LegalHold: { + Status: 'ON', + }, + }), + ), + ); + } + return Promise.all(putObjectLegalHolds) + .then(() => + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + BypassGovernanceRetention: true, + }), + ), + ) + .then(res => { + assert.strictEqual(res.Errors.length, 5); + res.Errors.forEach(err => assert.strictEqual(err.Code, 'AccessDenied')); + }); + }, + ); + it('should delete locked objects after retention period has expired', () => { const objects = createObjectsList(5, versionIds); const objectsCopy = JSON.parse(JSON.stringify(objects)); @@ -405,34 +467,44 @@ describe('Multi-Object Delete with Object Lock', () => { date: moment().subtract(10, 'days').toISOString(), }; return changeLockPromise(objectsCopy, newRetention) - .then(() => s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - }))).then(res => { - assert.strictEqual(res.Deleted.length, 5); - }).catch(err => { - checkNoError(err); - }); + .then(() => + s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ), + ) + .then(res => { + assert.strictEqual(res.Deleted.length, 5); + }) + .catch(err => { + checkNoError(err); + }); }); - it('should delete locked objects with GOVERNANCE ' + - 'retention mode and bypass header', () => { + it('should delete locked objects with GOVERNANCE ' + 'retention mode and bypass header', () => { const objects = createObjectsList(5, versionIds); - return s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - BypassGovernanceRetention: true, - })).then(res => { - assert.strictEqual(res.Deleted.length, 5); - assert.strictEqual(res.Errors, undefined); - }).catch(err => { - checkNoError(err); - }); + return s3 + .send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + BypassGovernanceRetention: true, + }), + ) + .then(res => { + assert.strictEqual(res.Deleted.length, 5); + assert.strictEqual(res.Errors, undefined); + }) + .catch(err => { + checkNoError(err); + }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/objectHead.js b/tests/functional/aws-node-sdk/test/object/objectHead.js index 3c92b2fcf2..742218804e 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead.js @@ -37,8 +37,7 @@ const objectName = 'someObject'; const partSize = 1024 * 1024 * 5; // 5MB minumum required part size. function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function dateFromNow(diff) { @@ -62,78 +61,84 @@ describe('HEAD object, conditions', () => { before(() => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - return bucketUtil.empty(bucketName).then(() => - bucketUtil.deleteOne(bucketName) - ) - .catch(err => { - if (err.name !== 'NoSuchBucket') { - process.stdout.write(`${err}\n`); - throw err; - } - }) - .then(() => bucketUtil.createOne(bucketName)); + return bucketUtil + .empty(bucketName) + .then(() => bucketUtil.deleteOne(bucketName)) + .catch(err => { + if (err.name !== 'NoSuchBucket') { + process.stdout.write(`${err}\n`); + throw err; + } + }) + .then(() => bucketUtil.createOne(bucketName)); }); function requestHead(fields, cb) { - s3.send(new HeadObjectCommand(Object.assign({ - Bucket: bucketName, - Key: objectName, - }, fields))).then(res => cb(null, res)).catch(cb); + s3.send( + new HeadObjectCommand( + Object.assign( + { + Bucket: bucketName, + Key: objectName, + }, + fields, + ), + ), + ) + .then(res => cb(null, res)) + .catch(cb); } - beforeEach(() => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'I am the best content ever', - })) - .then(res => { - etag = res.ETag; - etagTrim = etag.substring(1, etag.length - 1); - return s3.send(new HeadObjectCommand( - { Bucket: bucketName, Key: objectName })); - }).then(res => { - lastModified = res.LastModified; - })); + beforeEach(() => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'I am the best content ever', + }), + ) + .then(res => { + etag = res.ETag; + etagTrim = etag.substring(1, etag.length - 1); + return s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })); + }) + .then(res => { + lastModified = res.LastModified; + }), + ); afterEach(() => bucketUtil.empty(bucketName)); after(() => bucketUtil.deleteOne(bucketName)); - it('If-Match: returns no error when ETag match, with double quotes ' + - 'around ETag', - done => { - requestHead({ IfMatch: etag }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, with double quotes ' + 'around ETag', done => { + requestHead({ IfMatch: etag }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, with double ' + - 'quotes around ETag', - done => { - requestHead({ IfMatch: `non-matching,${etag}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, with double ' + 'quotes around ETag', done => { + requestHead({ IfMatch: `non-matching,${etag}` }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', - done => { - requestHead({ IfMatch: etagTrim }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', done => { + requestHead({ IfMatch: etagTrim }, err => { + checkNoError(err); + done(); }); + }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', - done => { - requestHead({ IfMatch: `non-matching,${etagTrim}` }, err => { - checkNoError(err); - done(); - }); + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', done => { + requestHead({ IfMatch: `non-matching,${etagTrim}` }, err => { + checkNoError(err); + done(); }); + }); it('If-Match: returns no error when ETag match with *', done => { requestHead({ IfMatch: '*' }, err => { @@ -142,13 +147,12 @@ describe('HEAD object, conditions', () => { }); }); - it('If-Match: returns PreconditionFailed when ETag does not match', - done => { - requestHead({ IfMatch: 'non-matching ETag' }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + it('If-Match: returns PreconditionFailed when ETag does not match', done => { + requestHead({ IfMatch: 'non-matching ETag' }, err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); }); + }); it('If-None-Match: returns no error when ETag does not match', done => { requestHead({ IfNoneMatch: 'non-matching' }, err => { @@ -157,282 +161,320 @@ describe('HEAD object, conditions', () => { }); }); - it('If-None-Match: returns no error when all ETags do not match', - done => { - requestHead({ + it('If-None-Match: returns no error when all ETags do not match', done => { + requestHead( + { IfNoneMatch: 'non-matching,non-matching-either', - }, err => { + }, + err => { checkNoError(err); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when ETag match, with double ' + - 'quotes around ETag', - done => { - requestHead({ IfNoneMatch: etag }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, with double ' + 'quotes around ETag', done => { + requestHead({ IfNoneMatch: etag }, err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); }); + }); - it('If-None-Match: returns NotModified when one of ETags match, with ' + - 'double quotes around ETag', - done => { - requestHead({ + it('If-None-Match: returns NotModified when one of ETags match, with ' + 'double quotes around ETag', done => { + requestHead( + { IfNoneMatch: `non-matching,${etag}`, - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); - }); - - it('If-None-Match: returns NotModified when ETag match, without ' + - 'double quotes around ETag', - done => { - requestHead({ IfNoneMatch: etagTrim }, err => { + }, + err => { assert.equal(err.$metadata.httpStatusCode, 304); done(); - }); - }); + }, + ); + }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'without double quotes around ETag', - done => { - requestHead({ - IfNoneMatch: `non-matching,${etagTrim}`, - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + it('If-None-Match: returns NotModified when ETag match, without ' + 'double quotes around ETag', done => { + requestHead({ IfNoneMatch: etagTrim }, err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); }); + }); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'without double quotes around ETag', done => { - requestHead({ IfModifiedSince: dateFromNow(-1) }, + requestHead( + { + IfNoneMatch: `non-matching,${etagTrim}`, + }, err => { - checkNoError(err); + assert.equal(err.$metadata.httpStatusCode, 304); done(); - }); + }, + ); + }, + ); + + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', done => { + requestHead({ IfModifiedSince: dateFromNow(-1) }, err => { + checkNoError(err); + done(); }); + }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns NotModified if Last modified ' + - 'date is lesser', - done => { - requestHead({ IfModifiedSince: dateFromNow(1) }, - err => { - checkError(err, errorInstances.NotModified.code); - done(); - }); + it.skip('If-Modified-Since: returns NotModified if Last modified ' + 'date is lesser', done => { + requestHead({ IfModifiedSince: dateFromNow(1) }, err => { + checkError(err, errorInstances.NotModified.code); + done(); }); + }); - it('If-Modified-Since: returns NotModified if Last modified ' + - 'date is equal', - done => { - requestHead({ IfModifiedSince: dateConvert(lastModified) }, - err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + it('If-Modified-Since: returns NotModified if Last modified ' + 'date is equal', done => { + requestHead({ IfModifiedSince: dateConvert(lastModified) }, err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified date is ' + - 'greater', - done => { - requestHead({ IfUnmodifiedSince: dateFromNow(1) }, err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified date is ' + 'greater', done => { + requestHead({ IfUnmodifiedSince: dateFromNow(1) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', - done => { - requestHead({ IfUnmodifiedSince: dateConvert(lastModified) }, - err => { - checkNoError(err); - done(); - }); + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', done => { + requestHead({ IfUnmodifiedSince: dateConvert(lastModified) }, err => { + checkNoError(err); + done(); }); + }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', - done => { - requestHead({ IfUnmodifiedSince: dateFromNow(-1) }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', done => { + requestHead({ IfUnmodifiedSince: dateFromNow(-1) }, err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); }); + }); - it('If-Match & If-Unmodified-Since: returns no error when match Etag ' + - 'and lastModified is greater', + it( + 'If-Match & If-Unmodified-Since: returns no error when match Etag ' + 'and lastModified is greater', done => { - requestHead({ + requestHead( + { + IfMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); + }, + ); + + it('If-Match match & If-Unmodified-Since match', done => { + requestHead( + { IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { + IfUnmodifiedSince: dateFromNow(1), + }, + err => { checkNoError(err); done(); - }); - }); - - it('If-Match match & If-Unmodified-Since match', done => { - requestHead({ - IfMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + }, + ); }); it('If-Match not match & If-Unmodified-Since not match', done => { - requestHead({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('If-Match not match & If-Unmodified-Since match', done => { - requestHead({ - IfMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-Match match & If-Modified-Since not match', done => { - requestHead({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match match & If-Modified-Since match', done => { - requestHead({ - IfMatch: etagTrim, - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfMatch: etagTrim, + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since not match', done => { - requestHead({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('If-Match not match & If-Modified-Since match', done => { - requestHead({ - IfMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); - it('If-None-Match & If-Modified-Since: returns NotModified when Etag ' + - 'does not match and lastModified is greater', + it( + 'If-None-Match & If-Modified-Since: returns NotModified when Etag ' + + 'does not match and lastModified is greater', done => { - requestHead({ + requestHead( + { + IfNoneMatch: etagTrim, + IfModifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); + }, + ); + }, + ); + + it('If-None-Match not match & If-Modified-Since not match', done => { + requestHead( + { IfNoneMatch: etagTrim, - IfModifiedSince: dateFromNow(-1), - }, err => { + IfModifiedSince: dateFromNow(1), + }, + err => { assert.equal(err.$metadata.httpStatusCode, 304); done(); - }); - }); - - it('If-None-Match not match & If-Modified-Since not match', done => { - requestHead({ - IfNoneMatch: etagTrim, - IfModifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + }, + ); }); it('If-None-Match match & If-Modified-Since match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(-1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(-1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); // Skipping this test, because real AWS does not provide error as // expected it.skip('If-None-Match match & If-Modified-Since not match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfModifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfModifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(1), - }, err => { - checkNoError(err); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + checkNoError(err); + done(); + }, + ); }); it('If-None-Match match & If-Unmodified-Since not match', done => { - requestHead({ - IfNoneMatch: 'non-matching', - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfNoneMatch: 'non-matching', + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since match', done => { - requestHead({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 304); - done(); - }); + requestHead( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 304); + done(); + }, + ); }); it('If-None-Match not match & If-Unmodified-Since not match', done => { - requestHead({ - IfNoneMatch: etagTrim, - IfUnmodifiedSince: dateFromNow(-1), - }, err => { - assert.equal(err.$metadata.httpStatusCode, 412); - done(); - }); + requestHead( + { + IfNoneMatch: etagTrim, + IfUnmodifiedSince: dateFromNow(-1), + }, + err => { + assert.equal(err.$metadata.httpStatusCode, 412); + done(); + }, + ); }); it('WebsiteRedirectLocation is set & it appears in response', done => { @@ -448,8 +490,7 @@ describe('HEAD object, conditions', () => { }; s3.send(new PutObjectCommand(redirBktwBody)).then(() => { s3.send(new HeadObjectCommand(redirBkt)).then(data => { - assert.strictEqual(data.WebsiteRedirectLocation, - 'http://google.com'); + assert.strictEqual(data.WebsiteRedirectLocation, 'http://google.com'); done(); }); }); @@ -477,73 +518,102 @@ describe('HEAD object, conditions', () => { it('WebsiteRedirectLocation is not set & is absent', done => { requestHead({}, (err, data) => { checkNoError(err); - assert.strictEqual('WebsiteRedirectLocation' in data, - false, 'WebsiteRedirectLocation header is present.'); + assert.strictEqual( + 'WebsiteRedirectLocation' in data, + false, + 'WebsiteRedirectLocation header is present.', + ); done(); }); }); - it('PartNumber is set & PartsCount is absent because object is not ' + - 'multipart', done => { + it('PartNumber is set & PartsCount is absent because object is not ' + 'multipart', done => { requestHead({ PartNumber: 1 }, (err, data) => { assert.ifError(err); - assert.strictEqual('PartsCount' in data, false, - 'PartsCount header is present.'); + assert.strictEqual('PartsCount' in data, false, 'PartsCount header is present.'); done(); }); }); - it('PartNumber is set & PartsCount appears in response for ' + - 'multipart object', done => { + it('PartNumber is set & PartsCount appears in response for ' + 'multipart object', done => { const mpuKey = 'mpukey'; - async.waterfall([ - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucketName, - Key: mpuKey, - })).then(data => next(null, data)).catch(next), - (data, next) => { - const uploadId = data.UploadId; - s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: mpuKey, - UploadId: uploadId, - PartNumber: 1, - Body: Buffer.alloc(partSize).fill('a'), - })).then(data => next(null, uploadId, data.ETag)).catch(next); - }, - (uploadId, etagOne, next) => s3.send(new UploadPartCommand({ - Bucket: bucketName, - Key: mpuKey, - UploadId: uploadId, - PartNumber: 2, - Body: Buffer.alloc(partSize).fill('z'), - })).then(data => next(null, uploadId, etagOne, data.ETag)).catch(next), - (uploadId, etagOne, etagTwo, next) => - s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucketName, - Key: mpuKey, - UploadId: uploadId, - MultipartUpload: { - Parts: [{ - PartNumber: 1, - ETag: etagOne, - }, { - PartNumber: 2, - ETag: etagTwo, - }], + async.waterfall( + [ + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: bucketName, + Key: mpuKey, + }), + ) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + const uploadId = data.UploadId; + s3.send( + new UploadPartCommand({ + Bucket: bucketName, + Key: mpuKey, + UploadId: uploadId, + PartNumber: 1, + Body: Buffer.alloc(partSize).fill('a'), + }), + ) + .then(data => next(null, uploadId, data.ETag)) + .catch(next); }, - })).then(data => next(null, data)).catch(next), - ], err => { - assert.ifError(err); - s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: mpuKey, - PartNumber: 1, - })).then(data => { - assert.strictEqual(data.PartsCount, 2); - done(); - }); - }); + (uploadId, etagOne, next) => + s3 + .send( + new UploadPartCommand({ + Bucket: bucketName, + Key: mpuKey, + UploadId: uploadId, + PartNumber: 2, + Body: Buffer.alloc(partSize).fill('z'), + }), + ) + .then(data => next(null, uploadId, etagOne, data.ETag)) + .catch(next), + (uploadId, etagOne, etagTwo, next) => + s3 + .send( + new CompleteMultipartUploadCommand({ + Bucket: bucketName, + Key: mpuKey, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + PartNumber: 1, + ETag: etagOne, + }, + { + PartNumber: 2, + ETag: etagTwo, + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + assert.ifError(err); + s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: mpuKey, + PartNumber: 1, + }), + ).then(data => { + assert.strictEqual(data.PartsCount, 2); + done(); + }); + }, + ); }); }); }); @@ -558,8 +628,8 @@ describe('HEAD object checksum mode', () => { const checksumAlgorithms = [ { algorithm: 'SHA256', responseField: 'ChecksumSHA256', internalName: 'sha256' }, - { algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' }, - { algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' }, + { algorithm: 'SHA1', responseField: 'ChecksumSHA1', internalName: 'sha1' }, + { algorithm: 'CRC32', responseField: 'ChecksumCRC32', internalName: 'crc32' }, { algorithm: 'CRC32C', responseField: 'ChecksumCRC32C', internalName: 'crc32c' }, { algorithm: 'CRC64NVME', responseField: 'ChecksumCRC64NVME', internalName: 'crc64nvme' }, ]; @@ -575,8 +645,7 @@ describe('HEAD object checksum mode', () => { for (const { internalName } of checksumAlgorithms) { // algorithms[internalName].digest() returns a base64 string - expectedDigests[internalName] = - await algorithms[internalName].digest(body); + expectedDigests[internalName] = await algorithms[internalName].digest(body); } }); @@ -587,56 +656,71 @@ describe('HEAD object checksum mode', () => { checksumAlgorithms.forEach(({ algorithm, responseField, internalName }) => { it(`should return ${responseField} and ChecksumType when ChecksumMode is ENABLED`, async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - Body: body, - ChecksumAlgorithm: algorithm, - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + Body: body, + ChecksumAlgorithm: algorithm, + }), + ); const storedChecksum = putRes[responseField]; assert(storedChecksum, `Expected ${responseField} in PutObject response`); - const headRes = await s3.send(new HeadObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - ChecksumMode: 'ENABLED', - })); - assert.strictEqual(headRes[responseField], expectedDigests[internalName], - `${responseField} value mismatch`); + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + ChecksumMode: 'ENABLED', + }), + ); + assert.strictEqual( + headRes[responseField], + expectedDigests[internalName], + `${responseField} value mismatch`, + ); assert.strictEqual(headRes[responseField], storedChecksum); assert.strictEqual(headRes.ChecksumType, 'FULL_OBJECT'); }); }); it('should not return checksum headers when ChecksumMode is not set', async () => { - await s3.send(new PutObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - Body: body, - ChecksumAlgorithm: 'SHA256', - })); - - const headRes = await s3.send(new HeadObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - })); + await s3.send( + new PutObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + Body: body, + ChecksumAlgorithm: 'SHA256', + }), + ); + + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + }), + ); assert.strictEqual(headRes.ChecksumSHA256, undefined); assert.strictEqual(headRes.ChecksumType, undefined); }); it('should return an error when ChecksumMode is not ENABLED', async () => { - await s3.send(new PutObjectCommand({ - Bucket: checksumBucket, - Key: checksumKey, - Body: body, - })); - - await assert.rejects( - s3.send(new HeadObjectCommand({ + await s3.send( + new PutObjectCommand({ Bucket: checksumBucket, Key: checksumKey, - ChecksumMode: 'DISABLED', - })), + Body: body, + }), + ); + + await assert.rejects( + s3.send( + new HeadObjectCommand({ + Bucket: checksumBucket, + Key: checksumKey, + ChecksumMode: 'DISABLED', + }), + ), ); }); }); @@ -661,13 +745,15 @@ describe('HEAD object with object lock', () => { ObjectLockMode: mockMode, ObjectLockLegalHoldStatus: 'ON', }; - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new PutObjectCommand(params)); const res = await s3.send(new GetObjectCommand({ Bucket: bucket, Key: key })); - + versionId = res.VersionId; }); @@ -691,7 +777,7 @@ describe('HEAD object with object lock', () => { it('should return object lock headers if set on the object', async () => { const res = await s3.send(new HeadObjectCommand({ Bucket: bucket, Key: key })); assert.strictEqual(res.ObjectLockMode, mockMode); - const responseDate= formatDate(res.ObjectLockRetainUntilDate); + const responseDate = formatDate(res.ObjectLockRetainUntilDate); const expectedDate = formatDate(mockDate); assert.strictEqual(responseDate, expectedDate); assert.strictEqual(res.ObjectLockLegalHoldStatus, 'ON'); diff --git a/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js b/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js index b4603858aa..075114d0e9 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead_compatibleHeaders.js @@ -2,74 +2,72 @@ const assert = require('assert'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { PutObjectCommand , HeadObjectCommand } = require('@aws-sdk/client-s3'); +const { PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3'); const bucketName = 'objectheadtestheaders'; const objectName = 'someObject'; -describe('HEAD object, compatibility headers [Cache-Control, ' + - 'Content-Disposition, Content-Encoding, Expires]', () => { - withV4(sigCfg => { - let bucketUtil; - let s3; - const cacheControl = 'max-age=86400'; - const contentDisposition = 'attachment; filename="fname.ext";'; - const contentEncoding = 'gzip,aws-chunked'; - // AWS Node SDK requires Date object, ISO-8601 string, or - // a UNIX timestamp for Expires header - const expires = new Date(); +describe( + 'HEAD object, compatibility headers [Cache-Control, ' + 'Content-Disposition, Content-Encoding, Expires]', + () => { + withV4(sigCfg => { + let bucketUtil; + let s3; + const cacheControl = 'max-age=86400'; + const contentDisposition = 'attachment; filename="fname.ext";'; + const contentEncoding = 'gzip,aws-chunked'; + // AWS Node SDK requires Date object, ISO-8601 string, or + // a UNIX timestamp for Expires header + const expires = new Date(); - before(() => { - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return bucketUtil.empty(bucketName).then(() => - bucketUtil.deleteOne(bucketName) - ) - .catch(err => { - if (err.name !== 'NoSuchBucket') { - process.stdout.write(`${err}\n`); - throw err; - } - }) - .then(() => bucketUtil.createOne(bucketName)) - .then(() => { - const params = { - Bucket: bucketName, - Key: objectName, - CacheControl: cacheControl, - ContentDisposition: contentDisposition, - ContentEncoding: contentEncoding, - Expires: expires, - }; - return s3.send(new PutObjectCommand(params)); + before(() => { + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return bucketUtil + .empty(bucketName) + .then(() => bucketUtil.deleteOne(bucketName)) + .catch(err => { + if (err.name !== 'NoSuchBucket') { + process.stdout.write(`${err}\n`); + throw err; + } + }) + .then(() => bucketUtil.createOne(bucketName)) + .then(() => { + const params = { + Bucket: bucketName, + Key: objectName, + CacheControl: cacheControl, + ContentDisposition: contentDisposition, + ContentEncoding: contentEncoding, + Expires: expires, + }; + return s3.send(new PutObjectCommand(params)); + }); }); - }); - after(async () => { - process.stdout.write('deleting bucket'); - await bucketUtil.empty(bucketName); - await bucketUtil.deleteOne(bucketName); - }); + after(async () => { + process.stdout.write('deleting bucket'); + await bucketUtil.empty(bucketName); + await bucketUtil.deleteOne(bucketName); + }); - it('should return additional headers if specified in objectPUT ' + - 'request', done => { - s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) - .then(res => { - assert.strictEqual(res.CacheControl, - cacheControl); - assert.strictEqual(res.ContentDisposition, - contentDisposition); - // Should remove V4 streaming value 'aws-chunked' - // to be compatible with AWS behavior - assert.strictEqual(res.ContentEncoding, - 'gzip,'); - assert.strictEqual(res.Expires.toGMTString(), - expires.toGMTString()); - return done(); - }).catch(err => { - process.stdout.write(`Error on headObject: ${err}\n`); - return done(err); - }); + it('should return additional headers if specified in objectPUT ' + 'request', done => { + s3.send(new HeadObjectCommand({ Bucket: bucketName, Key: objectName })) + .then(res => { + assert.strictEqual(res.CacheControl, cacheControl); + assert.strictEqual(res.ContentDisposition, contentDisposition); + // Should remove V4 streaming value 'aws-chunked' + // to be compatible with AWS behavior + assert.strictEqual(res.ContentEncoding, 'gzip,'); + assert.strictEqual(res.Expires.toGMTString(), expires.toGMTString()); + return done(); + }) + .catch(err => { + process.stdout.write(`Error on headObject: ${err}\n`); + return done(err); + }); + }); }); - }); -}); + }, +); diff --git a/tests/functional/aws-node-sdk/test/object/objectHead_replication.js b/tests/functional/aws-node-sdk/test/object/objectHead_replication.js index 39beb5e388..c5c0dde653 100644 --- a/tests/functional/aws-node-sdk/test/object/objectHead_replication.js +++ b/tests/functional/aws-node-sdk/test/object/objectHead_replication.js @@ -2,14 +2,15 @@ const assert = require('assert'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { removeAllVersions, versioningEnabled } = - require('../../lib/utility/versioning-util'); -const { PutObjectCommand, +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); +const { + PutObjectCommand, HeadObjectCommand, - CreateBucketCommand, + CreateBucketCommand, DeleteBucketCommand, - PutBucketVersioningCommand, - PutBucketReplicationCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, + PutBucketReplicationCommand, +} = require('@aws-sdk/client-s3'); const sourceBucket = 'source-bucket'; const keyPrefix = 'test-prefix'; @@ -27,10 +28,12 @@ describe("Head object 'ReplicationStatus' value", () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: sourceBucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(done => { @@ -38,40 +41,43 @@ describe("Head object 'ReplicationStatus' value", () => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: sourceBucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: sourceBucket })) + .then(() => done()) + .catch(done); }); }); - it('should be `undefined` when there is no bucket replication config', - async () => await checkHeadObj(`${keyPrefix}-foobar`, undefined)); + it('should be `undefined` when there is no bucket replication config', async () => + await checkHeadObj(`${keyPrefix}-foobar`, undefined)); describe('With bucket replication config', () => { const role = process.env.S3_END_TO_END - ? 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource' - : 'arn:aws:iam::123456789012:role/src-resource'; + ? 'arn:aws:iam::123456789012:role/src-resource,arn:aws:iam::123456789012:role/dest-resource' + : 'arn:aws:iam::123456789012:role/src-resource'; beforeEach(async () => { - await s3.send(new PutBucketReplicationCommand({ - Bucket: sourceBucket, - ReplicationConfiguration: { - Role: role, - Rules: [ - { - Destination: { StorageClass: 'us-east-2', - Bucket: 'arn:aws:s3:::dest-bucket' }, - Prefix: keyPrefix, - Status: 'Enabled', - }, - ], - }, - })); + await s3.send( + new PutBucketReplicationCommand({ + Bucket: sourceBucket, + ReplicationConfiguration: { + Role: role, + Rules: [ + { + Destination: { StorageClass: 'us-east-2', Bucket: 'arn:aws:s3:::dest-bucket' }, + Prefix: keyPrefix, + Status: 'Enabled', + }, + ], + }, + }), + ); }); - it("should be 'PENDING' when object key prefix applies", - async () => await checkHeadObj(`${keyPrefix}-foobar`, 'PENDING')); + it("should be 'PENDING' when object key prefix applies", async () => + await checkHeadObj(`${keyPrefix}-foobar`, 'PENDING')); - it('should be `undefined` when object key prefix does not apply', - async () => await checkHeadObj(`foobar-${keyPrefix}`, undefined)); + it('should be `undefined` when object key prefix does not apply', async () => + await checkHeadObj(`foobar-${keyPrefix}`, undefined)); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js index 177339898c..7efc12400e 100644 --- a/tests/functional/aws-node-sdk/test/object/objectOverwrite.js +++ b/tests/functional/aws-node-sdk/test/object/objectOverwrite.js @@ -56,7 +56,6 @@ const secondPutMetadata = { secondputagain: 'secondValue', }; - describe('Put object with same key as prior object', () => { withV4(sigCfg => { let bucketUtil; @@ -71,16 +70,20 @@ describe('Put object with same key as prior object', () => { }); beforeEach(async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'I am the best content ever', - Metadata: firstPutMetadata, - })); - const res = await s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'I am the best content ever', + Metadata: firstPutMetadata, + }), + ); + const res = await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(res.Metadata, firstPutMetadata); }); @@ -88,22 +91,25 @@ describe('Put object with same key as prior object', () => { after(async () => await bucketUtil.deleteOne(bucketName)); - it('should overwrite all user metadata and data on overwrite put', - async () => { - await s3.send(new PutObjectCommand({ + it('should overwrite all user metadata and data on overwrite put', async () => { + await s3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectName, Body: 'Much different', Metadata: secondPutMetadata, - })); - const res = await s3.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - assert.deepStrictEqual(res.Metadata, secondPutMetadata); - const bodyText = await res.Body.transformToString(); - assert.deepStrictEqual(bodyText, 'Much different'); - }); + }), + ); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + assert.deepStrictEqual(res.Metadata, secondPutMetadata); + const bodyText = await res.Body.transformToString(); + assert.deepStrictEqual(bodyText, 'Much different'); + }); coldStateScenarios.forEach(({ name, transitionInProgress, archiveState }) => { it(`should replace object with cold-state metadata (${name}) in non-versioned bucket`, async () => { @@ -113,12 +119,14 @@ describe('Put object with same key as prior object', () => { await fakeMetadataArchive(bucketName, objectName, undefined, archiveState); } - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: `overwrite cold state ${name}`, - Metadata: secondPutMetadata, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: `overwrite cold state ${name}`, + Metadata: secondPutMetadata, + }), + ); const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); @@ -129,17 +137,21 @@ describe('Put object with same key as prior object', () => { it('should create a new version when replacing archived current object in versioned bucket', async () => { await bucketUtil.empty(bucketName); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - - const firstPutRes = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'versioned first payload', - Metadata: firstPutMetadata, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + + const firstPutRes = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'versioned first payload', + Metadata: firstPutMetadata, + }), + ); assert(firstPutRes.VersionId); await fakeMetadataArchive(bucketName, objectName, undefined, { @@ -148,19 +160,23 @@ describe('Put object with same key as prior object', () => { restoreRequestedDays: 5, }); - const secondPutRes = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'versioned second payload', - Metadata: secondPutMetadata, - })); + const secondPutRes = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'versioned second payload', + Metadata: secondPutMetadata, + }), + ); assert(secondPutRes.VersionId); assert.notStrictEqual(secondPutRes.VersionId, firstPutRes.VersionId); - const headRes = await s3.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: objectName, - })); + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(headRes.Metadata, secondPutMetadata); const currentMD = await getMetadata(bucketName, objectName, undefined); @@ -170,25 +186,33 @@ describe('Put object with same key as prior object', () => { it('should replace archived current null version in version-suspended bucket', async () => { await bucketUtil.empty(bucketName); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'enabled-version-payload', - })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Suspended' }, - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'null-current-before-archive', - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'enabled-version-payload', + }), + ); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'null-current-before-archive', + }), + ); await fakeMetadataArchive(bucketName, objectName, undefined, { archiveInfo: { archiveId: 'archive-null-current' }, @@ -196,17 +220,18 @@ describe('Put object with same key as prior object', () => { restoreRequestedDays: 5, }); - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - Body: 'replace archived null current', - Metadata: secondPutMetadata, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + Body: 'replace archived null current', + Metadata: secondPutMetadata, + }), + ); const currentMD = await getMetadata(bucketName, objectName, undefined); assert.strictEqual(currentMD.archive, undefined); assert.deepStrictEqual(currentMD['x-amz-meta-secondput'], secondPutMetadata.secondput); }); - }); }); diff --git a/tests/functional/aws-node-sdk/test/object/put.js b/tests/functional/aws-node-sdk/test/object/put.js index aa1514c87d..124f217a14 100644 --- a/tests/functional/aws-node-sdk/test/object/put.js +++ b/tests/functional/aws-node-sdk/test/object/put.js @@ -2,7 +2,8 @@ const assert = require('assert'); const fs = require('fs'); const path = require('path'); const util = require('util'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, GetObjectAclCommand, GetObjectTaggingCommand, @@ -13,10 +14,8 @@ const BucketUtility = require('../../lib/utility/bucket-util'); const checkError = require('../../lib/utility/checkError'); const provideRawOutput = require('../../lib/utility/provideRawOutput'); const provideRawOutputAsync = util.promisify(provideRawOutput); -const { taggingTests, generateMultipleTagQuery } - = require('../../lib/utility/tagging'); -const genMaxSizeMetaHeaders - = require('../../lib/utility/genMaxSizeMetaHeaders'); +const { taggingTests, generateMultipleTagQuery } = require('../../lib/utility/tagging'); +const genMaxSizeMetaHeaders = require('../../lib/utility/genMaxSizeMetaHeaders'); const changeObjectLock = require('../../../../utilities/objectLock-util'); const object = 'object2putstuffin'; @@ -35,131 +34,144 @@ describe('PUT object', () => { afterEach(() => { process.stdout.write('Emptying bucket'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); - it('should put an object and set the acl via query param', - async () => { - // Create a temporary file for upload - const tempFile = path.join(__dirname, 'temp-upload-file.txt'); - fs.writeFileSync(tempFile, 'test content for upload'); - const params = { Bucket: bucket, Key: 'key', - ACL: 'public-read', StorageClass: 'STANDARD' }; - - const command = new PutObjectCommand(params); - const url = await getSignedUrl(s3, command); - const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, - '--upload-file', tempFile]); - fs.unlinkSync(tempFile); - assert.strictEqual(httpCode, '200 OK'); - const result = await s3.send(new GetObjectAclCommand({ Bucket: bucket, Key: 'key' })); - assert.deepStrictEqual(result.Grants[1], { Grantee: - { Type: 'Group', URI: - 'http://acs.amazonaws.com/groups/global/AllUsers', - }, Permission: 'READ' }); + it('should put an object and set the acl via query param', async () => { + // Create a temporary file for upload + const tempFile = path.join(__dirname, 'temp-upload-file.txt'); + fs.writeFileSync(tempFile, 'test content for upload'); + const params = { Bucket: bucket, Key: 'key', ACL: 'public-read', StorageClass: 'STANDARD' }; + + const command = new PutObjectCommand(params); + const url = await getSignedUrl(s3, command); + const { httpCode } = await provideRawOutputAsync(['-verbose', '-X', 'PUT', url, '--upload-file', tempFile]); + fs.unlinkSync(tempFile); + assert.strictEqual(httpCode, '200 OK'); + const result = await s3.send(new GetObjectAclCommand({ Bucket: bucket, Key: 'key' })); + assert.deepStrictEqual(result.Grants[1], { + Grantee: { Type: 'Group', URI: 'http://acs.amazonaws.com/groups/global/AllUsers' }, + Permission: 'READ', }); + }); - it('should put an object with key slash', - done => { - const params = { Bucket: bucket, Key: '/' }; - s3.send(new PutObjectCommand(params)).then(() => { + it('should put an object with key slash', done => { + const params = { Bucket: bucket, Key: '/' }; + s3.send(new PutObjectCommand(params)) + .then(() => { done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); done(err); }); - }); + }); it('should return KeyTooLong error when key is longer than 915 bytes', done => { const params = { Bucket: bucket, Key: 'a'.repeat(916) }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert(err, 'Expected error but did not find one'); - assert.strictEqual(err.name, 'KeyTooLong'); - assert.match(err.message, /915/); - done(); - }); + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert(err, 'Expected error but did not find one'); + assert.strictEqual(err.name, 'KeyTooLong'); + assert.match(err.message, /915/); + done(); + }); }); - it('should return error if putting object w/ > 2KB user-defined md', - done => { - const metadata = genMaxSizeMetaHeaders(); - const params = { Bucket: bucket, Key: '/', Metadata: metadata }; - s3.send(new PutObjectCommand(params)).then(() => { + it('should return error if putting object w/ > 2KB user-defined md', done => { + const metadata = genMaxSizeMetaHeaders(); + const params = { Bucket: bucket, Key: '/', Metadata: metadata }; + s3.send(new PutObjectCommand(params)) + .then(() => { // add one more byte to be over the limit metadata.header0 = `${metadata.header0}${'0'}`; - s3.send(new PutObjectCommand(params)).then(() => { + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert(err, 'Expected err but did not find one'); + assert.strictEqual(err.name, 'MetadataTooLarge'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(err); + }); + }); + + it( + 'should return InvalidRequest error if putting object with ' + + 'object lock retention date and mode when object lock is not ' + + 'enabled on the bucket', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key', + ObjectLockRetainUntilDate: date, + ObjectLockMode: 'GOVERNANCE', + }; + s3.send(new PutObjectCommand(params)) + .then(() => { assert(false, 'Expected failure but got success'); - }).catch(err => { - assert(err, 'Expected err but did not find one'); - assert.strictEqual(err.name, 'MetadataTooLarge'); + }) + .catch(err => { + const expectedErrMessage = 'Bucket is missing ObjectLockConfiguration'; + assert.strictEqual(err.name, 'InvalidRequest'); assert.strictEqual(err.$metadata.httpStatusCode, 400); + assert(err.toString().includes(expectedErrMessage)); done(); }); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(err); - }); - }); + }, + ); - it('should return InvalidRequest error if putting object with ' + - 'object lock retention date and mode when object lock is not ' + - 'enabled on the bucket', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key', - ObjectLockRetainUntilDate: date, - ObjectLockMode: 'GOVERNANCE', - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - const expectedErrMessage - = 'Bucket is missing ObjectLockConfiguration'; - assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - assert(err.toString().includes(expectedErrMessage)); - done(); - }); - }); - - it('should return Not Implemented error for obj. encryption using ' + - 'customer-provided encryption keys', done => { - const params = { Bucket: bucket, Key: 'key', - SSECustomerAlgorithm: 'AES256' }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'NotImplemented'); - done(); - }); - }); + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + done => { + const params = { Bucket: bucket, Key: 'key', SSECustomerAlgorithm: 'AES256' }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NotImplemented'); + done(); + }); + }, + ); - it('should return InvalidRedirectLocation if putting object ' + - 'with x-amz-website-redirect-location header that does not start ' + - 'with \'http://\', \'https://\' or \'/\'', done => { - const params = { Bucket: bucket, Key: 'key', - WebsiteRedirectLocation: 'google.com' }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidRedirectLocation'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + it( + 'should return InvalidRedirectLocation if putting object ' + + 'with x-amz-website-redirect-location header that does not start ' + + "with 'http://', 'https://' or '/'", + done => { + const params = { Bucket: bucket, Key: 'key', WebsiteRedirectLocation: 'google.com' }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidRedirectLocation'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); describe('Put object with tag set', () => { taggingTests.forEach(taggingTest => { @@ -167,153 +179,157 @@ describe('PUT object', () => { const key = encodeURIComponent(taggingTest.tag.key); const value = encodeURIComponent(taggingTest.tag.value); const tagging = `${key}=${value}`; - const params = { Bucket: bucket, Key: object, - Tagging: tagging }; - s3.send(new PutObjectCommand(params)).then(() => - s3.send(new GetObjectTaggingCommand({ Bucket: bucket, - Key: object })).then(data => { - assert.deepStrictEqual(data.TagSet[0], { - Key: taggingTest.tag.key, - Value: taggingTest.tag.value }); - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - })).catch(err => { - if (taggingTest.error) { - checkError(err, taggingTest.error, 400); + const params = { Bucket: bucket, Key: object, Tagging: tagging }; + s3.send(new PutObjectCommand(params)) + .then(() => + s3 + .send(new GetObjectTaggingCommand({ Bucket: bucket, Key: object })) + .then(data => { + assert.deepStrictEqual(data.TagSet[0], { + Key: taggingTest.tag.key, + Value: taggingTest.tag.value, + }); + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }), + ) + .catch(err => { + if (taggingTest.error) { + checkError(err, taggingTest.error, 400); + return done(); + } + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); return done(); - } - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - return done(); + }); }); }); - }); - it('should be able to put object with 10 tags', - done => { + it('should be able to put object with 10 tags', done => { const taggingConfig = generateMultipleTagQuery(10); - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: taggingConfig })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: taggingConfig })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); it('should be able to put an empty Tag set', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '', - })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: '' })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); - it('should be able to put object with empty tags', - done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '&&&&&&&&&&&&&&&&&key1=value1' })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + it('should be able to put object with empty tags', done => { + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: '&&&&&&&&&&&&&&&&&key1=value1' })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); it('should allow putting 50 tags', done => { const taggingConfig = generateMultipleTagQuery(50); - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: taggingConfig })).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: taggingConfig })) + .then(() => { + done(); + }) + .catch(err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }); }); - it('should return BadRequest if putting more that 50 tags', - done => { + it('should return BadRequest if putting more that 50 tags', done => { const taggingConfig = generateMultipleTagQuery(51); - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: taggingConfig })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'BadRequest', 400); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: taggingConfig })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'BadRequest', 400); + done(); + }); }); - it('should return InvalidArgument if using the same key twice', - done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: 'key1=value1&key1=value2' })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + it('should return InvalidArgument if using the same key twice', done => { + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: 'key1=value1&key1=value2' })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); - it('should return InvalidArgument if using the same key twice ' + - 'and empty tags', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2' })).then(() => { - assert(false, 'Expected failure but got success'); - - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + it('should return InvalidArgument if using the same key twice ' + 'and empty tags', done => { + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: object, + Tagging: '&&&&&&&&&&&&&&&&&key1=value1&key1=value2', + }), + ) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); it('should return InvalidArgument if tag with no key', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, - Tagging: '=value1', - })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: '=value1' })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); - it('should return InvalidArgument putting object with ' + - 'bad encoded tags', done => { - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: - 'key1==value1' })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + it('should return InvalidArgument putting object with ' + 'bad encoded tags', done => { + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: 'key1==value1' })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); - it('should return InvalidArgument putting object tag with ' + - 'invalid characters: %', done => { + it('should return InvalidArgument putting object tag with ' + 'invalid characters: %', done => { const value = 'value1%'; - s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: - `key1=${value}` })).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - checkError(err, 'InvalidArgument', 400); - done(); - }); + s3.send(new PutObjectCommand({ Bucket: bucket, Key: object, Tagging: `key1=${value}` })) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + checkError(err, 'InvalidArgument', 400); + done(); + }); }); }); }); }); - describe('PUT object with object lock', () => { const bucket = 'bucket2putstuffin4324242-lock'; withV4(sigCfg => { @@ -323,54 +339,61 @@ describe('PUT object with object lock', () => { beforeEach(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); }); afterEach(() => { process.stdout.write('Emptying bucket'); - return bucketUtil.empty(bucket) - .then(() => { - process.stdout.write('Deleting bucket'); - return bucketUtil.deleteOne(bucket); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .empty(bucket) + .then(() => { + process.stdout.write('Deleting bucket'); + return bucketUtil.deleteOne(bucket); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); - it('should put object with valid object lock retention date and ' + - 'mode when object lock is enabled on the bucket', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key1', - ObjectLockRetainUntilDate: date, - ObjectLockMode: 'COMPLIANCE', - }; - s3.send(new PutObjectCommand(params)).then(res => { - changeObjectLock( - [{ bucket, key: 'key1', versionId: res.VersionId }], '', done); - }); - }); + it( + 'should put object with valid object lock retention date and ' + + 'mode when object lock is enabled on the bucket', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key1', + ObjectLockRetainUntilDate: date, + ObjectLockMode: 'COMPLIANCE', + }; + s3.send(new PutObjectCommand(params)).then(res => { + changeObjectLock([{ bucket, key: 'key1', versionId: res.VersionId }], '', done); + }); + }, + ); - it('should put object with valid object lock retention date and ' + - 'mode when object lock is enabled on the bucket', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key2', - ObjectLockRetainUntilDate: date, - ObjectLockMode: 'GOVERNANCE', - }; - s3.send(new PutObjectCommand(params)).then(res => { - changeObjectLock( - [{ bucket, key: 'key2', versionId: res.VersionId }], '', done); - }); - }); + it( + 'should put object with valid object lock retention date and ' + + 'mode when object lock is enabled on the bucket', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key2', + ObjectLockRetainUntilDate: date, + ObjectLockMode: 'GOVERNANCE', + }; + s3.send(new PutObjectCommand(params)).then(res => { + changeObjectLock([{ bucket, key: 'key2', versionId: res.VersionId }], '', done); + }); + }, + ); it('should error with invalid object lock mode header', done => { const date = new Date(2050, 10, 10); @@ -380,13 +403,15 @@ describe('PUT object with object lock', () => { ObjectLockMode: 'Governance', ObjectLockRetainUntilDate: date, }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes('Unknown wormMode directive')); - done(); - }); + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes('Unknown wormMode directive')); + done(); + }); }); it('should put object with valid legal hold status ON', done => { @@ -396,8 +421,7 @@ describe('PUT object with object lock', () => { ObjectLockLegalHoldStatus: 'ON', }; s3.send(new PutObjectCommand(params)).then(res => { - changeObjectLock( - [{ bucket, key: 'key4', versionId: res.VersionId }], '', done); + changeObjectLock([{ bucket, key: 'key4', versionId: res.VersionId }], '', done); }); }); @@ -418,68 +442,83 @@ describe('PUT object with object lock', () => { Key: 'key6', ObjectLockLegalHoldStatus: 'on', }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes('Legal hold status must be one of "ON", "OFF"')); - done(); - }); + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes('Legal hold status must be one of "ON", "OFF"')); + done(); + }); }); - it('should return error when object lock retain until date header is ' + - 'provided but object lock mode header is missing', done => { - const date = new Date(2050, 10, 10); - const params = { - Bucket: bucket, - Key: 'key7', - ObjectLockRetainUntilDate: date, - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - const expectedErrMessage - = 'x-amz-object-lock-retain-until-date and ' + - 'x-amz-object-lock-mode must both be supplied'; - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes(expectedErrMessage)); - done(); - }); - }); + it( + 'should return error when object lock retain until date header is ' + + 'provided but object lock mode header is missing', + done => { + const date = new Date(2050, 10, 10); + const params = { + Bucket: bucket, + Key: 'key7', + ObjectLockRetainUntilDate: date, + }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + const expectedErrMessage = + 'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-mode must both be supplied'; + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes(expectedErrMessage)); + done(); + }); + }, + ); - it('should return error when object lock mode header is provided ' + - 'but object lock retain until date header is missing', done => { - const params = { - Bucket: bucket, - Key: 'key8', - ObjectLockMode: 'GOVERNANCE', - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - const expectedErrMessage - = 'x-amz-object-lock-retain-until-date and ' + - 'x-amz-object-lock-mode must both be supplied'; - assert.strictEqual(err.name, 'InvalidArgument'); - assert(err.toString().includes(expectedErrMessage)); - done(); - }); - }); + it( + 'should return error when object lock mode header is provided ' + + 'but object lock retain until date header is missing', + done => { + const params = { + Bucket: bucket, + Key: 'key8', + ObjectLockMode: 'GOVERNANCE', + }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + const expectedErrMessage = + 'x-amz-object-lock-retain-until-date and ' + 'x-amz-object-lock-mode must both be supplied'; + assert.strictEqual(err.name, 'InvalidArgument'); + assert(err.toString().includes(expectedErrMessage)); + done(); + }); + }, + ); - it('should return InvalidStorageClass error when x-amz-storage-class header is provided ' + - 'and not equal to STANDARD', done => { - const params = { - Bucket: bucket, - Key: 'key8', - StorageClass: 'COLD', - }; - s3.send(new PutObjectCommand(params)).then(() => { - assert(false, 'Expected failure but got success'); - }).catch(err => { - assert.strictEqual(err.name, 'InvalidStorageClass'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - done(); - }); - }); + it( + 'should return InvalidStorageClass error when x-amz-storage-class header is provided ' + + 'and not equal to STANDARD', + done => { + const params = { + Bucket: bucket, + Key: 'key8', + StorageClass: 'COLD', + }; + s3.send(new PutObjectCommand(params)) + .then(() => { + assert(false, 'Expected failure but got success'); + }) + .catch(err => { + assert.strictEqual(err.name, 'InvalidStorageClass'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + done(); + }); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/putObjAcl.js b/tests/functional/aws-node-sdk/test/object/putObjAcl.js index aa8ce0d9a7..a44a6b02cd 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjAcl.js +++ b/tests/functional/aws-node-sdk/test/object/putObjAcl.js @@ -1,15 +1,11 @@ const assert = require('assert'); -const { - PutObjectCommand, - PutObjectAclCommand, -} = require('@aws-sdk/client-s3'); +const { PutObjectCommand, PutObjectAclCommand } = require('@aws-sdk/client-s3'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); const constants = require('../../../../../constants'); -const notOwnerCanonicalID = '79a59df900b949e55d96a1e698fba' + - 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; +const notOwnerCanonicalID = '79a59df900b949e55d96a1e698fba' + 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; class _AccessControlPolicy { @@ -66,31 +62,32 @@ describe('PUT Object ACL', () => { it('should put object ACLs', async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; - const objects = [ - { Bucket, Key }, - ]; + const objects = [{ Bucket, Key }]; for (const param of objects) { await s3.send(new PutObjectCommand(param)); } - const data = await s3.send(new PutObjectAclCommand({ - Bucket, - Key, - ACL: 'public-read' - })); + const data = await s3.send( + new PutObjectAclCommand({ + Bucket, + Key, + ACL: 'public-read', + }), + ); assert(data); - }); + }); - it('should return NoSuchKey if try to put object ACLs ' + - 'for nonexistent object', async () => { + it('should return NoSuchKey if try to put object ACLs ' + 'for nonexistent object', async () => { const s3 = bucketUtil.s3; const Bucket = bucketName; try { - await s3.send(new PutObjectAclCommand({ - Bucket, - Key, - ACL: 'public-read' - })); + await s3.send( + new PutObjectAclCommand({ + Bucket, + Key, + ACL: 'public-read', + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { assert(err); @@ -103,25 +100,23 @@ describe('PUT Object ACL', () => { before(async () => { await s3.send(new PutObjectCommand({ Bucket: bucketName, Key })); }); - + after(async () => { process.stdout.write('deleting bucket'); await bucketUtil.empty(bucketName); }); - + // The supplied canonical ID is not associated with a real AWS // account, so AWS_ON_AIR will raise a 400 InvalidArgument - itSkipIfAWS('should return AccessDenied if try to change owner ' + - 'ID in ACL request body', async () => { - const acp = new _AccessControlPolicy( - { ownerID: notOwnerCanonicalID }); + itSkipIfAWS('should return AccessDenied if try to change owner ' + 'ID in ACL request body', async () => { + const acp = new _AccessControlPolicy({ ownerID: notOwnerCanonicalID }); acp.addGrantee('Group', constants.publicId, 'READ'); const putAclParams = { Bucket: bucketName, Key, AccessControlPolicy: acp, }; - + try { await s3.send(new PutObjectAclCommand(putAclParams)); throw new Error('Expected AccessDenied error'); diff --git a/tests/functional/aws-node-sdk/test/object/putObjTagging.js b/tests/functional/aws-node-sdk/test/object/putObjTagging.js index a86e089ca4..f4b6ac37ab 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjTagging.js +++ b/tests/functional/aws-node-sdk/test/object/putObjTagging.js @@ -16,11 +16,14 @@ const bucketName = 'testputtaggingbucket'; const objectName = 'testputtaggingobject'; const objectNameAcl = 'testputtaggingobjectacl'; -const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }; +const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], +}; function generateMultipleTagConfig(number) { const tags = []; @@ -61,28 +64,29 @@ describe('PUT object taggings', () => { taggingTests.forEach(taggingTest => { it(taggingTest.it, async () => { - const taggingConfig = generateTaggingConfig( - taggingTest.tag.key, - taggingTest.tag.value - ); - + const taggingConfig = generateTaggingConfig(taggingTest.tag.key, taggingTest.tag.value); + if (taggingTest.error) { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); assert.fail('Expected an error but request succeeded'); } catch (err) { checkError(err, taggingTest.error, 400); } } else { - const data = await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + const data = await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); assert.strictEqual(Object.keys(data).length, 1); } }); @@ -90,59 +94,70 @@ describe('PUT object taggings', () => { it('should allow putting 50 tags', async () => { const taggingConfig = generateMultipleTagConfig(50); - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); }); it('should return BadRequest if putting more than 50 tags', async () => { const taggingConfig = generateMultipleTagConfig(51); try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); assert.fail('Expected BadRequest error'); } catch (err) { checkError(err, 'BadRequest', 400); } }); - it('should put tag set', async () => { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); - const data = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); + const data = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); }); it('should return InvalidTag if using the same key twice', async () => { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }, - { - Key: 'key1', - Value: 'value2', + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + { + Key: 'key1', + Value: 'value2', + }, + ], }, - ] }, - })); + }), + ); throw new Error('Expected InvalidRequest error'); } catch (err) { checkError(err, 'InvalidTag', 400); @@ -151,18 +166,20 @@ describe('PUT object taggings', () => { it('should return InvalidTag if key is an empty string', async () => { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { - TagSet: [ - { - Key: '', - Value: 'value1', - }, - ] - } - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: '', + Value: 'value1', + }, + ], + }, + }), + ); assert.fail('Expected InvalidTag error'); } catch (err) { checkError(err, 'InvalidTag', 400); @@ -170,103 +187,126 @@ describe('PUT object taggings', () => { }); it('should be able to put an empty Tag set', async () => { - const data = await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [] } - })); + const data = await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { TagSet: [] }, + }), + ); assert.strictEqual(data.$metadata.httpStatusCode, 200); }); - it('should return NoSuchKey put tag to a non-existing object', - async () => { + it('should return NoSuchKey put tag to a non-existing object', async () => { try { - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: 'nonexisting', - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: 'nonexisting', + Tagging: taggingConfig, + }), + ); throw new Error('Expected NoSuchKey error'); } catch (err) { checkError(err, 'NoSuchKey', 404); } }); - it('should return 403 AccessDenied putting tag with another account', - async () => { + it('should return 403 AccessDenied putting tag with another account', async () => { try { - await otherAccountS3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); + await otherAccountS3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); throw new Error('Expected AccessDenied error'); } catch (err) { checkError(err, 'AccessDenied', 403); } }); - it('should return 403 AccessDenied putting tag with a different ' + - 'account to an object with ACL "public-read-write"', - async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - ACL: 'public-read-write', - })); + it( + 'should return 403 AccessDenied putting tag with a different ' + + 'account to an object with ACL "public-read-write"', + async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + ACL: 'public-read-write', + }), + ); - try { - await otherAccountS3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: taggingConfig, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); + try { + await otherAccountS3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: taggingConfig, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); - it('should return 403 AccessDenied putting tag to an object ' + - ' in a bucket created with a different account', - async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + it( + 'should return 403 AccessDenied putting tag to an object ' + + ' in a bucket created with a different account', + async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectNameAcl, + }), + ); - try { - await otherAccountS3.send(new PutObjectTaggingCommand({ + try { + await otherAccountS3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + Tagging: taggingConfig, + }), + ); + throw new Error('Expected AccessDenied error'); + } catch (err) { + checkError(err, 'AccessDenied', 403); + } + }, + ); + + it('should put tag to an object in a bucket created with same ' + 'account', async () => { + await s3.send( + new PutBucketAclCommand({ + Bucket: bucketName, + ACL: 'public-read-write', + }), + ); + await otherAccountS3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectNameAcl, - Tagging: taggingConfig, - })); - throw new Error('Expected AccessDenied error'); - } catch (err) { - checkError(err, 'AccessDenied', 403); - } - }); - - it('should put tag to an object in a bucket created with same ' + - 'account', async () => { - await s3.send(new PutBucketAclCommand({ - Bucket: bucketName, - ACL: 'public-read-write', - })); - await otherAccountS3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectNameAcl, - })); + }), + ); - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectNameAcl, - Tagging: taggingConfig, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectNameAcl, + Tagging: taggingConfig, + }), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js b/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js index 7dc9a2c23c..a263844a5f 100644 --- a/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js +++ b/tests/functional/aws-node-sdk/test/object/putObjectLegalHold.js @@ -48,7 +48,6 @@ function createLegalHoldParams(bucket, key, status, versionId) { return params; } - describe('PUT object legal hold', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -58,10 +57,12 @@ describe('PUT object legal hold', () => { let versionId; beforeEach(async () => { - await s3.send(new CreateBucketCommand({ - Bucket: bucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucket, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: key })); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); @@ -76,71 +77,83 @@ describe('PUT object legal hold', () => { resolve(); }); }) - .then(() => bucketUtil.empty(bucket, true)) - .then(() => bucketUtil.empty(unlockedBucket, true)) - .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])); + .then(() => bucketUtil.empty(bucket, true)) + .then(() => bucketUtil.empty(unlockedBucket, true)) + .then(() => bucketUtil.deleteMany([bucket, unlockedBucket])); }); - it('should return AccessDenied putting legal hold with another account', - done => { + it('should return AccessDenied putting legal hold with another account', done => { const params = createLegalHoldParams(bucket, key, 'ON'); - otherAccountS3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected AccessDenied error'); - }).catch(err => { - checkError(err, 'AccessDenied', 403); - done(); - }); + otherAccountS3 + .send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected AccessDenied error'); + }) + .catch(err => { + checkError(err, 'AccessDenied', 403); + done(); + }); }); it('should return NoSuchKey error if key does not exist', done => { const params = createLegalHoldParams(bucket, 'keynotexist', 'ON'); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected NoSuchKey error'); - }).catch(err => { - checkError(err, 'NoSuchKey', 404); - done(); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected NoSuchKey error'); + }) + .catch(err => { + checkError(err, 'NoSuchKey', 404); + done(); + }); }); it('should return NoSuchVersion error if version does not exist', done => { - s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - VersionId: '012345678901234567890123456789012', - LegalHold: mockLegalHold.on, - })).then(() => { - throw new Error('Expected NoSuchVersion error'); - }).catch(err => { - checkError(err, 'NoSuchVersion', 404); - done(); - }); + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + VersionId: '012345678901234567890123456789012', + LegalHold: mockLegalHold.on, + }), + ) + .then(() => { + throw new Error('Expected NoSuchVersion error'); + }) + .catch(err => { + checkError(err, 'NoSuchVersion', 404); + done(); + }); }); - it('should return InvalidRequest error putting legal hold to object ' + - 'in bucket with no object lock enabled', done => { - const params = createLegalHoldParams(unlockedBucket, key, 'ON'); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected InvalidRequest error'); - }).catch(err => { - checkError(err, 'InvalidRequest', 400); - done(); - }); - }); + it( + 'should return InvalidRequest error putting legal hold to object ' + + 'in bucket with no object lock enabled', + done => { + const params = createLegalHoldParams(unlockedBucket, key, 'ON'); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected InvalidRequest error'); + }) + .catch(err => { + checkError(err, 'InvalidRequest', 400); + done(); + }); + }, + ); - it('should return MethodNotAllowed if object version is delete marker', - done => { + it('should return MethodNotAllowed if object version is delete marker', done => { s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })) - .then(() => { - const params = createLegalHoldParams(bucket, key, 'ON'); - return s3.send(new PutObjectLegalHoldCommand(params)); - }) - .then(() => { - throw new Error('Expected MethodNotAllowed error'); - }) - .catch(err => { - checkError(err, 'MethodNotAllowed', 405); - done(); - }); + .then(() => { + const params = createLegalHoldParams(bucket, key, 'ON'); + return s3.send(new PutObjectLegalHoldCommand(params)); + }) + .then(() => { + throw new Error('Expected MethodNotAllowed error'); + }) + .catch(err => { + checkError(err, 'MethodNotAllowed', 405); + done(); + }); }); it('should put object legal hold ON', done => { @@ -150,55 +163,64 @@ describe('PUT object legal hold', () => { }); }); - it('should put object legal hold OFF', done => { const params = createLegalHoldParams(bucket, key, 'OFF'); s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - changeObjectLock([{ bucket, key, versionId }], '', done); + changeObjectLock([{ bucket, key, versionId }], '', done); }); }); it('should return error if request has empty or undefined Status', done => { const params = createLegalHoldParams(bucket, key, ''); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected MalformedXML error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected MalformedXML error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('should return error if request does not contain Status', done => { - s3.send(new PutObjectLegalHoldCommand({ - Bucket: bucket, - Key: key, - LegalHold: {}, - })).then(() => { - throw new Error('Expected MalformedXML error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: bucket, + Key: key, + LegalHold: {}, + }), + ) + .then(() => { + throw new Error('Expected MalformedXML error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('expects params.LegalHold.Status to be a string', done => { const params = createLegalHoldParams(bucket, key, true); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected InvalidParameterType error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected InvalidParameterType error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('expects Status request xml must be one of "ON", "OFF"', done => { const params = createLegalHoldParams(bucket, key, 'on'); - s3.send(new PutObjectLegalHoldCommand(params)).then(() => { - throw new Error('Expected MalformedXML error'); - }).catch(err => { - checkError(err, 'MalformedXML', 400); - changeObjectLock([{ bucket, key, versionId }], '', done); - }); + s3.send(new PutObjectLegalHoldCommand(params)) + .then(() => { + throw new Error('Expected MalformedXML error'); + }) + .catch(err => { + checkError(err, 'MalformedXML', 400); + changeObjectLock([{ bucket, key, versionId }], '', done); + }); }); it('should support request with versionId parameter', done => { @@ -232,7 +254,8 @@ describe('PUT object legal hold iam action and version id', () => { const unauthBucketUtil = new BucketUtility('default', sigCfg, true); const unauthS3 = unauthBucketUtil.s3; const CommandClass = eval(operation); - unauthS3.send(new CommandClass(params)) + unauthS3 + .send(new CommandClass(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } @@ -254,18 +277,21 @@ describe('PUT object legal hold iam action and version id', () => { beforeEach(() => { process.stdout.write('Setting up bucket policy legal hold tests\n'); - return s3.send(new CreateBucketCommand({ - Bucket: testBucket, - ObjectLockEnabledForBucket: true, - })) - .then(() => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: key }))) - .then(res => { - versionId = res.VersionId; - }) - .catch(err => { - process.stdout.write('Error in beforeEach\n'); - throw err; - }); + return s3 + .send( + new CreateBucketCommand({ + Bucket: testBucket, + ObjectLockEnabledForBucket: true, + }), + ) + .then(() => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: key }))) + .then(res => { + versionId = res.VersionId; + }) + .catch(err => { + process.stdout.write('Error in beforeEach\n'); + throw err; + }); }); afterEach(async () => { @@ -300,30 +326,41 @@ describe('PUT object legal hold iam action and version id', () => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBucket, - Policy: JSON.stringify(bucketPolicy), - })).then(() => { - done(); - }).catch(err => { - assert.ifError(err); - done(); - }); + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBucket, + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.ifError(err); + done(); + }); }); if (testCase.expectedResult === 'allow') { afterEach(() => - s3.send(new PutObjectLegalHoldCommand({ - Bucket: testBucket, - Key: key, - LegalHold: { Status: 'OFF' }, - })) - .then(() => s3.send(new PutObjectLegalHoldCommand({ - Bucket: testBucket, - Key: key, - VersionId: versionId, - LegalHold: { Status: 'OFF' }, - }))) + s3 + .send( + new PutObjectLegalHoldCommand({ + Bucket: testBucket, + Key: key, + LegalHold: { Status: 'OFF' }, + }), + ) + .then(() => + s3.send( + new PutObjectLegalHoldCommand({ + Bucket: testBucket, + Key: key, + VersionId: versionId, + LegalHold: { Status: 'OFF' }, + }), + ), + ), ); } diff --git a/tests/functional/aws-node-sdk/test/object/putPart.js b/tests/functional/aws-node-sdk/test/object/putPart.js index ec66956a03..17ca70e5d9 100644 --- a/tests/functional/aws-node-sdk/test/object/putPart.js +++ b/tests/functional/aws-node-sdk/test/object/putPart.js @@ -21,50 +21,56 @@ describe('PUT object', () => { beforeEach(async () => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - + await s3.send(new CreateBucketCommand({ Bucket: bucket })); - const res = await s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key - })); + const res = await s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ); uploadId = res.UploadId; }); afterEach(async () => { - await s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - })); + await s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ); await bucketUtil.empty(bucket); await bucketUtil.deleteOne(bucket); }); - it('should return Not Implemented error for obj. encryption using ' + - 'customer-provided encryption keys', async () => { - const params = { - Bucket: bucket, - Key: 'key', - PartNumber: 0, - UploadId: uploadId, - SSECustomerAlgorithm: 'AES256' - }; - try { - await s3.send(new UploadPartCommand(params)); - throw new Error('Expected NotImplemented error'); - } catch (err) { - assert.strictEqual(err.name, 'NotImplemented'); - } - }); + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + async () => { + const params = { + Bucket: bucket, + Key: 'key', + PartNumber: 0, + UploadId: uploadId, + SSECustomerAlgorithm: 'AES256', + }; + try { + await s3.send(new UploadPartCommand(params)); + throw new Error('Expected NotImplemented error'); + } catch (err) { + assert.strictEqual(err.name, 'NotImplemented'); + } + }, + ); it('should return InvalidArgument if negative PartNumber', async () => { const params = { Bucket: bucket, Key: 'key', PartNumber: -1, - UploadId: uploadId + UploadId: uploadId, }; - + try { await s3.send(new UploadPartCommand(params)); assert.fail('Expected InvalidArgument error'); diff --git a/tests/functional/aws-node-sdk/test/object/putRetention.js b/tests/functional/aws-node-sdk/test/object/putRetention.js index a01d163724..9a8e940945 100644 --- a/tests/functional/aws-node-sdk/test/object/putRetention.js +++ b/tests/functional/aws-node-sdk/test/object/putRetention.js @@ -6,7 +6,7 @@ const { PutObjectCommand, DeleteObjectCommand, PutObjectRetentionCommand, - PutBucketPolicyCommand + PutBucketPolicyCommand, } = require('@aws-sdk/client-s3'); const { errorInstances } = require('arsenal'); @@ -24,7 +24,6 @@ const retentionConfig = { RetainUntilDate: moment().add(1, 'd').add(123, 'ms').toDate(), }; - const changeObjectLockPromise = promisify(changeObjectLock); describe('PUT object retention', () => { @@ -36,10 +35,12 @@ describe('PUT object retention', () => { let versionId; beforeEach(async () => { - await s3.send(new CreateBucketCommand({ - Bucket: bucketName, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: bucketName, + ObjectLockEnabledForBucket: true, + }), + ); await s3.send(new CreateBucketCommand({ Bucket: unlockedBucket })); await s3.send(new PutObjectCommand({ Bucket: unlockedBucket, Key: objectName })); const putRes = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: objectName })); @@ -54,11 +55,13 @@ describe('PUT object retention', () => { it('should return AccessDenied putting retention with another account', async () => { try { - await otherAccountS3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await otherAccountS3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'AccessDenied', 403); @@ -67,11 +70,13 @@ describe('PUT object retention', () => { it('should return NoSuchKey error if key does not exist', async () => { try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: 'thiskeydoesnotexist', - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: 'thiskeydoesnotexist', + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchKey', 404); @@ -80,40 +85,48 @@ describe('PUT object retention', () => { it('should return NoSuchVersion error if version does not exist', async () => { try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: '012345678901234567890123456789012', - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: '012345678901234567890123456789012', + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchVersion', 404); } }); - it('should return InvalidRequest error putting retention to object in bucket with no object lock ' + - 'enabled', async () => { - try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: unlockedBucket, - Key: objectName, - Retention: retentionConfig, - })); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'InvalidRequest', 400); - } - }); + it( + 'should return InvalidRequest error putting retention to object in bucket with no object lock ' + 'enabled', + async () => { + try { + await s3.send( + new PutObjectRetentionCommand({ + Bucket: unlockedBucket, + Key: objectName, + Retention: retentionConfig, + }), + ); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'InvalidRequest', 400); + } + }, + ); it('should return MethodNotAllowed if object version is delete marker', async () => { await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: objectName })); try { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'MethodNotAllowed', 405); @@ -121,21 +134,25 @@ describe('PUT object retention', () => { }); it('should put object retention', async () => { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + }), + ); await changeObjectLockPromise([{ bucket: bucketName, key: objectName, versionId }], ''); }); it('should support request with versionId parameter', async () => { - await s3.send(new PutObjectRetentionCommand({ - Bucket: bucketName, - Key: objectName, - Retention: retentionConfig, - VersionId: versionId, - })); + await s3.send( + new PutObjectRetentionCommand({ + Bucket: bucketName, + Key: objectName, + Retention: retentionConfig, + VersionId: versionId, + }), + ); await changeObjectLockPromise([{ bucket: bucketName, key: objectName, versionId }], ''); }); }); @@ -161,13 +178,13 @@ describe('PUT object retention iam action and version id', () => { const unauthBucketUtil = new BucketUtility('default', sigCfg, true); const unauthS3 = unauthBucketUtil.s3; const CommandClass = eval(operation); - unauthS3.send(new CommandClass(params)) + unauthS3 + .send(new CommandClass(params)) .then(data => callback(null, data)) .catch(err => callback(err)); } } - function cbNoError(done) { return err => { assert.ifError(err); @@ -183,10 +200,12 @@ describe('PUT object retention iam action and version id', () => { } beforeEach(async () => { - await s3.send(new CreateBucketCommand({ - Bucket: testBucket, - ObjectLockEnabledForBucket: true, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: testBucket, + ObjectLockEnabledForBucket: true, + }), + ); const res = await s3.send(new PutObjectCommand({ Bucket: testBucket, Key: objectName })); versionId = res.VersionId; }); @@ -223,15 +242,19 @@ describe('PUT object retention iam action and version id', () => { Version: '2012-10-17', Statement: [statement], }; - s3.send(new PutBucketPolicyCommand({ - Bucket: testBucket, - Policy: JSON.stringify(bucketPolicy), - })).then(() => { - done(); - }).catch(err => { - assert.ifError(err); - done(); - }); + s3.send( + new PutBucketPolicyCommand({ + Bucket: testBucket, + Policy: JSON.stringify(bucketPolicy), + }), + ) + .then(() => { + done(); + }) + .catch(err => { + assert.ifError(err); + done(); + }); }); it(`should ${testCase.expectedResult} unauthenticated putObjectRetention without VersionId`, done => { diff --git a/tests/functional/aws-node-sdk/test/object/putVersion.js b/tests/functional/aws-node-sdk/test/object/putVersion.js index dcb7aa4eee..d49e869d40 100644 --- a/tests/functional/aws-node-sdk/test/object/putVersion.js +++ b/tests/functional/aws-node-sdk/test/object/putVersion.js @@ -8,20 +8,20 @@ const { DummyRequestLogger } = require('../../../../unit/helpers'); const checkError = require('../../lib/utility/checkError'); const { getMetadata, fakeMetadataArchive, isNullKeyMetadataV1 } = require('../utils/init'); const { hasColdStorage } = require('../../lib/utility/test-utils'); -const { CreateBucketCommand, - PutObjectCommand, - HeadObjectCommand, +const { + CreateBucketCommand, + PutObjectCommand, + HeadObjectCommand, GetObjectCommand, PutObjectAclCommand, PutObjectTaggingCommand, PutObjectLegalHoldCommand, ListObjectsCommand, DeleteObjectCommand, - PutBucketVersioningCommand } = require('@aws-sdk/client-s3'); + PutBucketVersioningCommand, +} = require('@aws-sdk/client-s3'); -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const log = new DummyRequestLogger(); const bucketName = 'bucket1putversion32'; @@ -46,7 +46,7 @@ function putObjectVersion(s3, params, vid, cb) { { step: 'build', name: 'addVersionIdHeader', // Add a name to identify the middleware - } + }, ); const promise = s3.send(command); @@ -67,7 +67,6 @@ function clearRestoreStatus(versions) { return versions; } - function checkVersionsAndUpdate(versionsBefore, versionsAfter, indexes) { indexes.forEach(i => { assert.notStrictEqual(versionsAfter[i].value.Size, versionsBefore[i].value.Size); @@ -98,16 +97,22 @@ describe('PUT object with x-scal-s3-version-id header', () => { beforeEach(done => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - async.series([ - next => metadata.setup(next), - next => s3.send(new CreateBucketCommand({ Bucket: bucketName })).then(() => { - next(); - }), - next => s3.send(new CreateBucketCommand({ Bucket: bucketNameMD, - ObjectLockEnabledForBucket: true })).then(() => { - next(); - }), - ], done); + async.series( + [ + next => metadata.setup(next), + next => + s3.send(new CreateBucketCommand({ Bucket: bucketName })).then(() => { + next(); + }), + next => + s3 + .send(new CreateBucketCommand({ Bucket: bucketNameMD, ObjectLockEnabledForBucket: true })) + .then(() => { + next(); + }), + ], + done, + ); }); afterEach(async () => { @@ -121,35 +126,43 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => putObjectVersion(s3, params, 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P', err => { - assert.strictEqual(err.name, 'InvalidArgument'); - assert.strictEqual(err.$metadata.httpStatusCode, 400); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + putObjectVersion(s3, params, 'aJLWKz4Ko9IjBBgXKj5KQT.G9UHv0g7P', err => { + assert.strictEqual(err.name, 'InvalidArgument'); + assert.strictEqual(err.$metadata.httpStatusCode, 400); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if key does not exist', done => { const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => putObjectVersion(s3, params, '', err => { - checkError(err, 'NoSuchKey', 404); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => + putObjectVersion(s3, params, '', err => { + checkError(err, 'NoSuchKey', 404); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if version does not exist', done => { @@ -157,37 +170,49 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => putObjectVersion(s3, params, - '393833343735313131383832343239393939393952473030312020313031', err => { - checkError(err, 'NoSuchVersion', 404); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + putObjectVersion( + s3, + params, + '393833343735313131383832343239393939393952473030312020313031', + err => { + checkError(err, 'NoSuchVersion', 404); + return next(); + }, + ), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if archiving is not in progress', done => { const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => putObjectVersion(s3, params, '', err => { - checkError(err, 'InvalidObjectState', 403); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + putObjectVersion(s3, params, '', err => { + checkError(err, 'InvalidObjectState', 403); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); it('should fail if trying to overwrite a delete marker', done => { @@ -196,68 +221,89 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new DeleteObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => putObjectVersion(s3, params, vId, err => { - checkError(err, 'MethodNotAllowed', 405); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new DeleteObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => + putObjectVersion(s3, params, vId, err => { + checkError(err, 'MethodNotAllowed', 405); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); }); describeSkipNullMdV1('with cold storage location', () => { it('should overwrite an object', done => { - const params = { Bucket: bucketName, Key: objectName }; + const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; let objMDAfter; let versionsBefore; let versionsAfter; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => { - next(); - }), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'originOp', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => + s3.send(new PutObjectCommand(params)).then(() => { + next(); + }), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'originOp', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a version', done => { @@ -265,7 +311,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -274,41 +320,57 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current version if empty version id header', done => { @@ -316,7 +378,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -325,41 +387,57 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a non-current null version', done => { @@ -367,7 +445,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let versionsBefore; @@ -375,39 +453,54 @@ describe('PUT object with x-scal-s3-version-id header', () => { let objMDBefore; let objMDAfter; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, 'null', archive, next), - next => getMetadata(bucketName, objectName, 'null', (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - next(err); - }), - next => putObjectVersion(s3, params, 'null', next), - next => getMetadata(bucketName, objectName, 'null', (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, 'null', archive, next), + next => + getMetadata(bucketName, objectName, 'null', (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + next(err); + }), + next => putObjectVersion(s3, params, 'null', next), + next => + getMetadata(bucketName, objectName, 'null', (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the lastest version and keep nullVersionId', done => { @@ -415,7 +508,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let versionsBefore; @@ -424,42 +517,58 @@ describe('PUT object with x-scal-s3-version-id header', () => { let objMDAfter; let vId; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a current null version', done => { @@ -467,13 +576,13 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const sParams = { Bucket: bucketName, VersioningConfiguration: { Status: 'Suspended', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -481,40 +590,55 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsBefore; let versionsAfter; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutBucketVersioningCommand(sParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => s3.send(new PutBucketVersioningCommand(sParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite a non-current version', done => { @@ -522,7 +646,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -531,43 +655,59 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [1]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current version', done => { @@ -575,7 +715,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -584,42 +724,58 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current version after bucket version suspended', done => { @@ -627,13 +783,13 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const sParams = { Bucket: bucketName, VersioningConfiguration: { Status: 'Suspended', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -642,43 +798,63 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsAfter; let vId; - async.series([ - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => s3.send(new PutObjectCommand(params)).then(res => { - vId = res.VersionId; - return next(); - }), - next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => s3.send(new PutBucketVersioningCommand(sParams)).then(() => next()).catch(next), - next => putObjectVersion(s3, params, vId, next), - next => getMetadata(bucketName, objectName, vId, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => + s3.send(new PutObjectCommand(params)).then(res => { + vId = res.VersionId; + return next(); + }), + next => fakeMetadataArchive(bucketName, objectName, vId, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => + s3 + .send(new PutBucketVersioningCommand(sParams)) + .then(() => next()) + .catch(next), + next => putObjectVersion(s3, params, vId, next), + next => + getMetadata(bucketName, objectName, vId, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should overwrite the current null version after bucket version enabled', done => { @@ -686,7 +862,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { Bucket: bucketName, VersioningConfiguration: { Status: 'Enabled', - } + }, }; const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; @@ -694,38 +870,53 @@ describe('PUT object with x-scal-s3-version-id header', () => { let versionsBefore; let versionsAfter; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsBefore = clearRestoreStatus(res.Versions); - return next(err); - }), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), - next => putObjectVersion(s3, params, 'null', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, (err, res) => { - versionsAfter = clearRestoreStatus(res.Versions); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); - assert.deepStrictEqual(versionsAfter, versionsBefore); - - checkObjMdAndUpdate(objMDBefore, objMDAfter, ['location', 'content-length', 'originOp', - 'microVersionId', 'x-amz-restore', 'archive', 'dataStoreName', 'checksum']); - assert.deepStrictEqual(objMDAfter, objMDBefore); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsBefore = clearRestoreStatus(res.Versions); + return next(err); + }), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => s3.send(new PutBucketVersioningCommand(vParams)).then(() => next()), + next => putObjectVersion(s3, params, 'null', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + metadata.listObject(bucketName, mdListingParams, log, (err, res) => { + versionsAfter = clearRestoreStatus(res.Versions); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + checkVersionsAndUpdate(versionsBefore, versionsAfter, [0]); + assert.deepStrictEqual(versionsAfter, versionsBefore); + + checkObjMdAndUpdate(objMDBefore, objMDAfter, [ + 'location', + 'content-length', + 'originOp', + 'microVersionId', + 'x-amz-restore', + 'archive', + 'dataStoreName', + 'checksum', + ]); + assert.deepStrictEqual(objMDAfter, objMDBefore); + return done(); + }, + ); }); it('should fail if restore is already completed', done => { @@ -735,255 +926,299 @@ describe('PUT object with x-scal-s3-version-id header', () => { restoreRequestedAt: new Date(0), restoreRequestedDays: 5, restoreCompletedAt: new Date(10), - restoreWillExpireAt: new Date(10 + (5 * 24 * 60 * 60 * 1000)), + restoreWillExpireAt: new Date(10 + 5 * 24 * 60 * 60 * 1000), }; - async.series([ - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archiveCompleted, next), - next => putObjectVersion(s3, params, '', err => { - checkError(err, 'InvalidObjectState', 403); - return next(); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - return done(); - }); + async.series( + [ + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archiveCompleted, next), + next => + putObjectVersion(s3, params, '', err => { + checkError(err, 'InvalidObjectState', 403); + return next(); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + return done(); + }, + ); }); - [ - 'non versioned', - 'versioned', - 'suspended' - ].forEach(versioning => { + ['non versioned', 'versioned', 'suspended'].forEach(versioning => { it(`should update restore metadata while keeping storage class (${versioning})`, done => { const params = { Bucket: bucketName, Key: objectName }; let objMDBefore; let objMDAfter; - - async.series([ - next => { - if (versioning === 'versioned') { - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' } - })).then(() => next()); - } else if (versioning === 'suspended') { - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Suspended' } - })).then(() => next()); - } - return next(); + async.series( + [ + next => { + if (versioning === 'versioned') { + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()); + } else if (versioning === 'suspended') { + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(() => next()); + } + return next(); + }, + next => s3.send(new PutObjectCommand(params)).then(() => next()), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDBefore = objMD; + return next(err); + }), + next => metadata.listObject(bucketName, mdListingParams, log, err => next(err)), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + objMDAfter = objMD; + return next(err); + }), + next => + s3 + .send(new ListObjectsCommand({ Bucket: bucketName })) + .then(res => { + assert.strictEqual(res.Contents.length, 1); + assert.strictEqual(res.Contents[0].StorageClass, LOCATION_NAME_DMF); + return next(); + }) + .catch(err => { + assert.ifError(err); + return next(err); + }), + next => + s3 + .send(new HeadObjectCommand(params)) + .then(res => { + assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); + return next(); + }) + .catch(err => { + assert.ifError(err); + return next(err); + }), + next => + s3 + .send(new GetObjectCommand(params)) + .then(res => { + assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); + return next(); + }) + .catch(err => { + assert.ifError(err); + return next(err); + }), + ], + err => { + assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); + + // storage class must stay as the cold location + assert.deepStrictEqual(objMDAfter['x-amz-storage-class'], LOCATION_NAME_DMF); + + /// Make sure object data location is set back to its bucket data location. + assert.deepStrictEqual(objMDAfter.dataStoreName, 'us-east-1'); + + assert.deepStrictEqual(objMDAfter.archive.archiveInfo, objMDBefore.archive.archiveInfo); + assert.deepStrictEqual( + objMDAfter.archive.restoreRequestedAt, + objMDBefore.archive.restoreRequestedAt, + ); + assert.deepStrictEqual( + objMDAfter.archive.restoreRequestedDays, + objMDBefore.archive.restoreRequestedDays, + ); + assert.deepStrictEqual(objMDAfter['x-amz-restore']['ongoing-request'], false); + + assert(objMDAfter.archive.restoreCompletedAt); + assert(objMDAfter.archive.restoreWillExpireAt); + assert(objMDAfter['x-amz-restore']['expiry-date']); + return done(); }, - next => s3.send(new PutObjectCommand(params)).then(() => next()), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDBefore = objMD; - return next(err); - }), - next => metadata.listObject(bucketName, mdListingParams, log, err => next(err)), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - objMDAfter = objMD; - return next(err); - }), - next => s3.send(new ListObjectsCommand({ Bucket: bucketName })).then(res => { - assert.strictEqual(res.Contents.length, 1); - assert.strictEqual(res.Contents[0].StorageClass, LOCATION_NAME_DMF); - return next(); - }).catch(err => { - assert.ifError(err); - return next(err); - }), - next => s3.send(new HeadObjectCommand(params)).then(res => { - assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); - return next(); - }).catch(err => { - assert.ifError(err); - return next(err); - }), - next => s3.send(new GetObjectCommand(params)).then(res => { - assert.strictEqual(res.StorageClass, LOCATION_NAME_DMF); - return next(); - }).catch(err => { - assert.ifError(err); - return next(err); - }), - ], err => { - assert.strictEqual(err, null, `Expected success got error ${JSON.stringify(err)}`); - - // storage class must stay as the cold location - assert.deepStrictEqual(objMDAfter['x-amz-storage-class'], LOCATION_NAME_DMF); - - /// Make sure object data location is set back to its bucket data location. - assert.deepStrictEqual(objMDAfter.dataStoreName, 'us-east-1'); - - assert.deepStrictEqual(objMDAfter.archive.archiveInfo, objMDBefore.archive.archiveInfo); - assert.deepStrictEqual(objMDAfter.archive.restoreRequestedAt, - objMDBefore.archive.restoreRequestedAt); - assert.deepStrictEqual(objMDAfter.archive.restoreRequestedDays, - objMDBefore.archive.restoreRequestedDays); - assert.deepStrictEqual(objMDAfter['x-amz-restore']['ongoing-request'], false); - - assert(objMDAfter.archive.restoreCompletedAt); - assert(objMDAfter.archive.restoreWillExpireAt); - assert(objMDAfter['x-amz-restore']['expiry-date']); - return done(); - }); + ); }); }); it('should "copy" all but non data-related metadata (data encryption, data size...)', done => { - const params = { - Bucket: bucketNameMD, - Key: objectName - }; - const putParams = { - ...params, - Metadata: { - 'custom-user-md': 'custom-md', - }, - WebsiteRedirectLocation: 'http://custom-redirect' - }; - const aclParams = { - ...params, - // email of user Bart defined in authdata.json - GrantFullControl: 'emailaddress=sampleaccount1@sampling.com', - }; - const tagParams = { - ...params, - Tagging: { - TagSet: [{ + const params = { + Bucket: bucketNameMD, + Key: objectName, + }; + const putParams = { + ...params, + Metadata: { + 'custom-user-md': 'custom-md', + }, + WebsiteRedirectLocation: 'http://custom-redirect', + }; + const aclParams = { + ...params, + // email of user Bart defined in authdata.json + GrantFullControl: 'emailaddress=sampleaccount1@sampling.com', + }; + const tagParams = { + ...params, + Tagging: { + TagSet: [ + { Key: 'tag1', - Value: 'value1' - }, { + Value: 'value1', + }, + { Key: 'tag2', - Value: 'value2' - }] - } - }; - const legalHoldParams = { - ...params, - LegalHold: { - Status: 'ON' + Value: 'value2', }, - }; - const acl = { - 'Canned': '', - 'FULL_CONTROL': [ - // canonicalID of user Bart - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', - ], - 'WRITE_ACP': [], - 'READ': [], - 'READ_ACP': [], - }; - const tags = { tag1: 'value1', tag2: 'value2' }; - const replicationInfo = { - 'status': 'COMPLETED', - 'backends': [ - { - 'site': 'azure-normal', - 'status': 'COMPLETED', - 'dataStoreVersionId': '', - }, ], - 'content': [ - 'DATA', - 'METADATA', - ], - 'destination': 'arn:aws:s3:::versioned', - 'storageClass': 'azure-normal', - 'role': 'arn:aws:iam::root:role/s3-replication-role', - 'storageType': 'azure', - 'dataStoreVersionId': '', - 'isNFS': null, + }, + }; + const legalHoldParams = { + ...params, + LegalHold: { + Status: 'ON', + }, + }; + const acl = { + Canned: '', + FULL_CONTROL: [ + // canonicalID of user Bart + '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + ], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }; + const tags = { tag1: 'value1', tag2: 'value2' }; + const replicationInfo = { + status: 'COMPLETED', + backends: [ + { + site: 'azure-normal', + status: 'COMPLETED', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::versioned', + storageClass: 'azure-normal', + role: 'arn:aws:iam::root:role/s3-replication-role', + storageType: 'azure', + dataStoreVersionId: '', + isNFS: null, }; - async.series([ - next => s3.send(new PutObjectCommand(putParams)).then(() => next()), - next => s3.send(new PutObjectAclCommand(aclParams)).then(() => next()), - next => s3.send(new PutObjectTaggingCommand(tagParams)).then(() => next()), - next => s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()), - next => getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - /* eslint-disable no-param-reassign */ - objMD.dataStoreName = LOCATION_NAME_DMF; - objMD.archive = archive; - objMD.replicationInfo = replicationInfo; - // data related - objMD['content-length'] = 99; - objMD['content-type'] = 'testtype'; - objMD['content-md5'] = 'testmd5'; - objMD['content-encoding'] = 'testencoding'; - objMD['x-amz-server-side-encryption'] = 'aws:kms'; - /* eslint-enable no-param-reassign */ - return metadata.putObjectMD(bucketNameMD, objectName, objMD, undefined, log, next); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - assert.deepStrictEqual(objMD.acl, acl); - assert.deepStrictEqual(objMD.tags, tags); - assert.deepStrictEqual(objMD.replicationInfo, replicationInfo); - assert.deepStrictEqual(objMD.legalHold, true); - assert.strictEqual(objMD['x-amz-meta-custom-user-md'], 'custom-md'); - assert.strictEqual(objMD['x-amz-website-redirect-location'], 'http://custom-redirect'); - // make sure data related metadatas ar not the same before and after - assert.notStrictEqual(objMD['x-amz-server-side-encryption'], 'aws:kms'); - assert.notStrictEqual(objMD['content-length'], 99); - assert.notStrictEqual(objMD['content-encoding'], 'testencoding'); - assert.notStrictEqual(objMD['content-type'], 'testtype'); - // make sure we keep the same etag and add the new restored - // data's etag inside x-amz-restore - assert.strictEqual(objMD['content-md5'], 'testmd5'); - assert.strictEqual(typeof objMD['x-amz-restore']['content-md5'], 'string'); - return next(); - }), - // removing legal hold to be able to clean the bucket after the test - next => { - legalHoldParams.LegalHold.Status = 'OFF'; - return s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()); - }, - ], done); + async.series( + [ + next => s3.send(new PutObjectCommand(putParams)).then(() => next()), + next => s3.send(new PutObjectAclCommand(aclParams)).then(() => next()), + next => s3.send(new PutObjectTaggingCommand(tagParams)).then(() => next()), + next => s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()), + next => + getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + /* eslint-disable no-param-reassign */ + objMD.dataStoreName = LOCATION_NAME_DMF; + objMD.archive = archive; + objMD.replicationInfo = replicationInfo; + // data related + objMD['content-length'] = 99; + objMD['content-type'] = 'testtype'; + objMD['content-md5'] = 'testmd5'; + objMD['content-encoding'] = 'testencoding'; + objMD['x-amz-server-side-encryption'] = 'aws:kms'; + /* eslint-enable no-param-reassign */ + return metadata.putObjectMD(bucketNameMD, objectName, objMD, undefined, log, next); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketNameMD, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + assert.deepStrictEqual(objMD.acl, acl); + assert.deepStrictEqual(objMD.tags, tags); + assert.deepStrictEqual(objMD.replicationInfo, replicationInfo); + assert.deepStrictEqual(objMD.legalHold, true); + assert.strictEqual(objMD['x-amz-meta-custom-user-md'], 'custom-md'); + assert.strictEqual(objMD['x-amz-website-redirect-location'], 'http://custom-redirect'); + // make sure data related metadatas ar not the same before and after + assert.notStrictEqual(objMD['x-amz-server-side-encryption'], 'aws:kms'); + assert.notStrictEqual(objMD['content-length'], 99); + assert.notStrictEqual(objMD['content-encoding'], 'testencoding'); + assert.notStrictEqual(objMD['content-type'], 'testtype'); + // make sure we keep the same etag and add the new restored + // data's etag inside x-amz-restore + assert.strictEqual(objMD['content-md5'], 'testmd5'); + assert.strictEqual(typeof objMD['x-amz-restore']['content-md5'], 'string'); + return next(); + }), + // removing legal hold to be able to clean the bucket after the test + next => { + legalHoldParams.LegalHold.Status = 'OFF'; + return s3.send(new PutObjectLegalHoldCommand(legalHoldParams)).then(() => next()); + }, + ], + done, + ); }); it('should set restore originOp and drop restore-attempt metadata', done => { const params = { Bucket: bucketName, Key: objectName }; - async.series([ - next => s3.send(new PutObjectCommand({ - ...params, - Metadata: { - 'custom-md': 'preserved-value', - }, - })).then(() => next()).catch(next), - next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - /* eslint-disable no-param-reassign */ - objMD['x-amz-meta-scal-s3-restore-attempt'] = '3'; - /* eslint-enable no-param-reassign */ - return metadata.putObjectMD(bucketName, objectName, objMD, undefined, log, next); - }), - next => putObjectVersion(s3, params, '', next), - next => getMetadata(bucketName, objectName, undefined, (err, objMD) => { - if (err) { - return next(err); - } - assert.strictEqual(objMD.originOp, 's3:ObjectRestore:Completed'); - assert.strictEqual(objMD['x-amz-meta-custom-md'], 'preserved-value'); - assert.strictEqual(objMD['x-amz-meta-scal-s3-restore-attempt'], undefined); - return next(); - }), - ], done); + async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + ...params, + Metadata: { + 'custom-md': 'preserved-value', + }, + }), + ) + .then(() => next()) + .catch(next), + next => fakeMetadataArchive(bucketName, objectName, undefined, archive, next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + /* eslint-disable no-param-reassign */ + objMD['x-amz-meta-scal-s3-restore-attempt'] = '3'; + /* eslint-enable no-param-reassign */ + return metadata.putObjectMD(bucketName, objectName, objMD, undefined, log, next); + }), + next => putObjectVersion(s3, params, '', next), + next => + getMetadata(bucketName, objectName, undefined, (err, objMD) => { + if (err) { + return next(err); + } + assert.strictEqual(objMD.originOp, 's3:ObjectRestore:Completed'); + assert.strictEqual(objMD['x-amz-meta-custom-md'], 'preserved-value'); + assert.strictEqual(objMD['x-amz-meta-scal-s3-restore-attempt'], undefined); + return next(); + }), + ], + done, + ); }); it('should keep x-amz-meta-scal-version-id when restoring on ingestion bucket', async () => { @@ -991,12 +1226,14 @@ describe('PUT object with x-scal-s3-version-id header', () => { const params = { Bucket: ingestionBucketName, Key: objectName }; let putVersionId; try { - await s3.send(new CreateBucketCommand({ - Bucket: ingestionBucketName, - CreateBucketConfiguration: { - LocationConstraint: 'us-east-2:ingest', - }, - })); + await s3.send( + new CreateBucketCommand({ + Bucket: ingestionBucketName, + CreateBucketConfiguration: { + LocationConstraint: 'us-east-2:ingest', + }, + }), + ); const putRes = await s3.send(new PutObjectCommand(params)); putVersionId = putRes.VersionId; @@ -1005,8 +1242,7 @@ describe('PUT object with x-scal-s3-version-id header', () => { await putObjectVersion(s3, params, putVersionId); - const restoredObjMD = await getMetadata( - ingestionBucketName, objectName, putVersionId); + const restoredObjMD = await getMetadata(ingestionBucketName, objectName, putVersionId); assert.strictEqual(restoredObjMD['x-amz-meta-scal-version-id'], putVersionId); } finally { diff --git a/tests/functional/aws-node-sdk/test/object/rangeTest.js b/tests/functional/aws-node-sdk/test/object/rangeTest.js index 3e0a66b606..fa5a671aa1 100644 --- a/tests/functional/aws-node-sdk/test/object/rangeTest.js +++ b/tests/functional/aws-node-sdk/test/object/rangeTest.js @@ -32,8 +32,7 @@ function getOuterRange(range, bytes) { arr[1] = Number.parseInt(bytes, 10) - 1; } else { arr[0] = arr[0] === '' ? 0 : Number.parseInt(arr[0], 10); - arr[1] = arr[1] === '' || Number.parseInt(arr[1], 10) >= bytes ? - Number.parseInt(bytes, 10) - 1 : arr[1]; + arr[1] = arr[1] === '' || Number.parseInt(arr[1], 10) >= bytes ? Number.parseInt(bytes, 10) - 1 : arr[1]; } return { begin: arr[0], @@ -44,58 +43,69 @@ function getOuterRange(range, bytes) { // Get the ranged object from a bucket. Write the response body to a file, then // use getRangeExec to check that all the bytes are in the correct location. function checkRanges(range, bytes) { - return s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - Range: `bytes=${range}`, - })) - .then(async res => { - const { begin, end } = getOuterRange(range, bytes); - const total = (end - begin) + 1; - // If the range header is '-' (i.e., it is invalid), content range - // should be undefined - const contentRange = range === '-' ? undefined : - `bytes ${begin}-${end}/${bytes}`; + return s3 + .send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + Range: `bytes=${range}`, + }), + ) + .then(async res => { + const { begin, end } = getOuterRange(range, bytes); + const total = end - begin + 1; + // If the range header is '-' (i.e., it is invalid), content range + // should be undefined + const contentRange = range === '-' ? undefined : `bytes ${begin}-${end}/${bytes}`; - assert.deepStrictEqual(res.ContentLength, total); - assert.deepStrictEqual(res.ContentRange, contentRange); - assert(res.ContentType === undefined || - res.ContentType === 'application/octet-stream'); - assert.deepStrictEqual(res.Metadata, {}); + assert.deepStrictEqual(res.ContentLength, total); + assert.deepStrictEqual(res.ContentRange, contentRange); + assert(res.ContentType === undefined || res.ContentType === 'application/octet-stream'); + assert.deepStrictEqual(res.Metadata, {}); - const bodyBytes = await res.Body.transformToByteArray(); - const bodyBuffer = Buffer.from(bodyBytes); - await writeFileAsync(`hashedFile.${bytes}.${range}`, bodyBuffer); - return execFileAsync('./getRangeExec', ['--check', '--size', total, - '--offset', begin, `hashedFile.${bytes}.${range}`]); - }); + const bodyBytes = await res.Body.transformToByteArray(); + const bodyBuffer = Buffer.from(bodyBytes); + await writeFileAsync(`hashedFile.${bytes}.${range}`, bodyBuffer); + return execFileAsync('./getRangeExec', [ + '--check', + '--size', + total, + '--offset', + begin, + `hashedFile.${bytes}.${range}`, + ]); + }); } // Create 5MB parts and upload them as parts of a MPU. Returns array of part // responses (with ETag) for CompleteMultipartUpload. async function uploadParts(bytes, uploadId) { const name = `hashedFile.${bytes}`; - return Promise.all([1, 2].map(async part => { - try { - await execFileAsync('dd', [ - `if=${name}`, - `of=${name}.mpuPart${part}`, - 'bs=5242880', - `skip=${part - 1}`, - 'count=1', - ]); - const res = await s3.send(new UploadPartCommand({ - Bucket: bucket, - Key: key, - PartNumber: part, - UploadId: uploadId, - Body: createReadStream(`${name}.mpuPart${part}`), - })); - return res; - } catch (error) { - throw new Error(`Error uploading part ${part}: ${error.message}`); - } - })); + return Promise.all( + [1, 2].map(async part => { + try { + await execFileAsync('dd', [ + `if=${name}`, + `of=${name}.mpuPart${part}`, + 'bs=5242880', + `skip=${part - 1}`, + 'count=1', + ]); + const res = await s3.send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + PartNumber: part, + UploadId: uploadId, + Body: createReadStream(`${name}.mpuPart${part}`), + }), + ); + return res; + } catch (error) { + throw new Error(`Error uploading part ${part}: ${error.message}`); + } + }), + ); } // Create a hashed file of size bytes @@ -105,8 +115,7 @@ function createHashedFile(bytes) { } describe('aws-node-sdk range tests', () => { - before(() => execFileAsync('gcc', ['-o', 'getRangeExec', - 'lib/utility/getRange.c'])); + before(() => execFileAsync('gcc', ['-o', 'getRangeExec', 'lib/utility/getRange.c'])); after(() => execAsync('rm getRangeExec')); describe('aws-node-sdk range test for object put by MPU', () => @@ -117,65 +126,79 @@ describe('aws-node-sdk range tests', () => { let uploadId; beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => s3.send(new CreateMultipartUploadCommand({ - Bucket: bucket, - Key: key, - }))) - .then(res => { - uploadId = res.UploadId; - }) - .then(() => createHashedFile(fileSize)) - .then(() => uploadParts(fileSize, uploadId)) - .then(res => s3.send(new CompleteMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - MultipartUpload: { - Parts: [ - { - ETag: res[0].ETag, - PartNumber: 1, - }, - { - ETag: res[1].ETag, - PartNumber: 2, - }, - ], - }, - }))) + s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => + s3.send( + new CreateMultipartUploadCommand({ + Bucket: bucket, + Key: key, + }), + ), + ) + .then(res => { + uploadId = res.UploadId; + }) + .then(() => createHashedFile(fileSize)) + .then(() => uploadParts(fileSize, uploadId)) + .then(res => + s3.send( + new CompleteMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + MultipartUpload: { + Parts: [ + { + ETag: res[0].ETag, + PartNumber: 1, + }, + { + ETag: res[1].ETag, + PartNumber: 2, + }, + ], + }, + }), + ), + ), ); - afterEach(() => bucketUtil.empty(bucket) - .then(() => s3.send(new AbortMultipartUploadCommand({ - Bucket: bucket, - Key: key, - UploadId: uploadId, - }))) - .catch(err => { - // Upload was already completed in beforeEach; abort is no-op - if (err.name === 'NoSuchUpload' || err.code === 'NoSuchUpload') { - return; - } - throw err; - }) - .then(() => bucketUtil.deleteOne(bucket)) - .then(() => execAsync(`rm hashedFile.${fileSize}*`)) + afterEach(() => + bucketUtil + .empty(bucket) + .then(() => + s3.send( + new AbortMultipartUploadCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + }), + ), + ) + .catch(err => { + // Upload was already completed in beforeEach; abort is no-op + if (err.name === 'NoSuchUpload' || err.code === 'NoSuchUpload') { + return; + } + throw err; + }) + .then(() => bucketUtil.deleteOne(bucket)) + .then(() => execAsync(`rm hashedFile.${fileSize}*`)), ); - it('should get a range from the first part of an object', () => - checkRanges('0-9', fileSize)); + it('should get a range from the first part of an object', () => checkRanges('0-9', fileSize)); - it('should get a range from the second part of an object', () => - checkRanges('5242880-5242889', fileSize)); + it('should get a range from the second part of an object', () => checkRanges('5242880-5242889', fileSize)); - it('should get a range that spans both parts of an object', () => - checkRanges('5242875-5242884', fileSize)); + it('should get a range that spans both parts of an object', () => checkRanges('5242875-5242884', fileSize)); - it('should get a range from the second part of an object and ' + - 'include the end if the range requested goes beyond the ' + - 'actual object end', () => - checkRanges('10485750-10485790', fileSize)); + it( + 'should get a range from the second part of an object and ' + + 'include the end if the range requested goes beyond the ' + + 'actual object end', + () => checkRanges('10485750-10485790', fileSize), + ); })); describe('aws-node-sdk range test of regular object put (non-MPU)', () => @@ -185,18 +208,26 @@ describe('aws-node-sdk range tests', () => { const fileSize = 2000; beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => createHashedFile(fileSize)) - .then(() => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: createReadStream(`hashedFile.${fileSize}`), - })))); + s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => createHashedFile(fileSize)) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: createReadStream(`hashedFile.${fileSize}`), + }), + ), + ), + ); afterEach(() => - bucketUtil.empty(bucket) - .then(() => bucketUtil.deleteOne(bucket)) - .then(() => execAsync(`rm hashedFile.${fileSize}*`))); + bucketUtil + .empty(bucket) + .then(() => bucketUtil.deleteOne(bucket)) + .then(() => execAsync(`rm hashedFile.${fileSize}*`)), + ); const putRangeTests = [ '-', // Test for invalid range @@ -229,9 +260,9 @@ describe('aws-node-sdk range tests', () => { ]; putRangeTests.forEach(range => { - it(`should get a range of ${range} bytes using a ${fileSize} ` + - 'byte sized object', () => - checkRanges(range, fileSize)); + it(`should get a range of ${range} bytes using a ${fileSize} ` + 'byte sized object', () => + checkRanges(range, fileSize), + ); }); })); @@ -242,26 +273,36 @@ describe('aws-node-sdk range tests', () => { const fileSize = 2900; beforeEach(() => - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => createHashedFile(fileSize)) - .then(() => s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: createReadStream(`hashedFile.${fileSize}`), - })))); + s3 + .send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => createHashedFile(fileSize)) + .then(() => + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: createReadStream(`hashedFile.${fileSize}`), + }), + ), + ), + ); afterEach(() => - bucketUtil.empty(bucket) - .then(() => bucketUtil.deleteOne(bucket)) - .then(() => execAsync(`rm hashedFile.${fileSize}*`))); + bucketUtil + .empty(bucket) + .then(() => bucketUtil.deleteOne(bucket)) + .then(() => execAsync(`rm hashedFile.${fileSize}*`)), + ); - it('should get the final 90 bytes of a 2890 byte object for a ' + - 'byte range of 2800-', () => - checkRanges('2800-', fileSize)); + it('should get the final 90 bytes of a 2890 byte object for a ' + 'byte range of 2800-', () => + checkRanges('2800-', fileSize), + ); - it('should get the final 90 bytes of a 2890 byte object for a ' + - 'byte range of 2800-Number.MAX_SAFE_INTEGER', () => - checkRanges(`2800-${Number.MAX_SAFE_INTEGER}`, fileSize)); + it( + 'should get the final 90 bytes of a 2890 byte object for a ' + + 'byte range of 2800-Number.MAX_SAFE_INTEGER', + () => checkRanges(`2800-${Number.MAX_SAFE_INTEGER}`, fileSize), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteGet.js b/tests/functional/aws-node-sdk/test/object/websiteGet.js index e66144385f..39a3db6eb3 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteGet.js +++ b/tests/functional/aws-node-sdk/test/object/websiteGet.js @@ -22,32 +22,49 @@ const config = getConfig('default'); const s3Client = new S3Client(config); const s3 = { createBucket: (params, cb) => { - s3Client.send(new CreateBucketCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new CreateBucketCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, deleteBucket: (params, cb) => { - s3Client.send(new DeleteBucketCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new DeleteBucketCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, putBucketWebsite: (params, cb) => { - s3Client.send(new PutBucketWebsiteCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new PutBucketWebsiteCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, putObject: (params, cb) => { - s3Client.send(new PutObjectCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new PutObjectCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, deleteObject: (params, cb) => { - s3Client.send(new DeleteObjectCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new DeleteObjectCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, putBucketPolicy: (params, cb) => { - s3Client.send(new PutBucketPolicyCommand(params)).then(d => cb(null, d)).catch(cb); + s3Client + .send(new PutBucketPolicyCommand(params)) + .then(d => cb(null, d)) + .catch(cb); }, }; const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; const port = process.env.AWS_ON_AIR ? 80 : 8000; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; const endpoint = `${transport}://${hostname}:${port}`; const redirectEndpoint = `${transport}://www.google.com`; @@ -58,27 +75,33 @@ const redirectEndpoint = `${transport}://www.google.com`; function putBucketWebsiteAndPutObjectRedirect(redirect, condition, key, done) { const webConfig = new WebsiteConfigTester('index.html'); webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { if (err) { done(err); } - return s3.putObject({ Bucket: bucket, - Key: key, - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' }, done); + return s3.putObject( + { + Bucket: bucket, + Key: key, + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }, + done, + ); }); } describe('User visits bucket website endpoint', () => { it('should return 404 when no such bucket', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '404-no-such-bucket', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '404-no-such-bucket', + }, + done, + ); }); describe('with existing bucket', () => { @@ -87,122 +110,138 @@ describe('User visits bucket website endpoint', () => { afterEach(done => s3.deleteBucket({ Bucket: bucket }, done)); it('should return 404 when no website configuration', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '404-no-such-website-configuration', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '404-no-such-website-configuration', + }, + done, + ); }); describe('with existing configuration', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' }, + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, err => { assert.strictEqual(err, null); done(); - }); + }, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, - err => done(err)); - }); - - it('should return 405 when user requests method other than get ' + - 'or head', done => { - makeRequest({ - hostname, - port, - method: 'POST', - }, (err, res) => { - assert.strictEqual(err, null, - `Err with request ${err}`); - assert.strictEqual(res.statusCode, 405); - assert(res.body.indexOf('405 ' + - 'Method Not Allowed') > -1); - return done(); - }); + s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, err => done(err)); + }); + + it('should return 405 when user requests method other than get ' + 'or head', done => { + makeRequest( + { + hostname, + port, + method: 'POST', + }, + (err, res) => { + assert.strictEqual(err, null, `Err with request ${err}`); + assert.strictEqual(res.statusCode, 405); + assert(res.body.indexOf('405 ' + 'Method Not Allowed') > -1); + return done(); + }, + ); }); it('should serve indexDocument if no key requested', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'index-user', + }, + done, + ); }); it('should serve indexDocument if key requested', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/index.html`, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/index.html`, + responseType: 'index-user', + }, + done, + ); }); }); describe('with path in request with/without key', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, - Key: 'pathprefix/index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'pathprefix/index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, + done, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: - 'pathprefix/index.html' }, - done); + s3.deleteObject({ Bucket: bucket, Key: 'pathprefix/index.html' }, done); }); - it('should serve indexDocument if path request without key', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/pathprefix/`, - responseType: 'index-user', - }, done); + it('should serve indexDocument if path request without key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/pathprefix/`, + responseType: 'index-user', + }, + done, + ); }); - it('should serve indexDocument if path request with key', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/pathprefix/index.html`, - responseType: 'index-user', - }, done); + it('should serve indexDocument if path request with key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/pathprefix/index.html`, + responseType: 'index-user', + }, + done, + ); }); }); describe('with private key', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, - Key: 'index.html', - ACL: 'private', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'private', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, + done, + ); }); }); @@ -211,27 +250,32 @@ describe('User visits bucket website endpoint', () => { }); it('should return 403 if key is private', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-access-denied', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); }); }); describe('with nonexisting index document key', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); it('should return 403 if nonexisting index document key', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-access-denied', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); }); }); @@ -240,80 +284,91 @@ describe('User visits bucket website endpoint', () => { const redirectAllTo = { HostName: 'www.google.com', }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); it(`should redirect to ${redirectEndpoint}`, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/`, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/`, + }, + done, + ); }); it(`should redirect to ${redirectEndpoint}/about`, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about`, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about`, + }, + done, + ); }); }); - describe.skip('redirect all requests to https://www.google.com ' + - 'since https protocol set in website config', () => { - // Note: these tests will all redirect to https even if - // conf does not have https since protocol in website config - // specifies https - beforeEach(done => { - const redirectAllTo = { - HostName: 'www.google.com', - Protocol: 'https', - }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); - }); + describe.skip( + 'redirect all requests to https://www.google.com ' + 'since https protocol set in website config', + () => { + // Note: these tests will all redirect to https even if + // conf does not have https since protocol in website config + // specifies https + beforeEach(done => { + const redirectAllTo = { + HostName: 'www.google.com', + Protocol: 'https', + }; + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); + }); - it('should redirect to https://google.com/', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: 'https://www.google.com/', - }, done); - }); + it('should redirect to https://google.com/', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: 'https://www.google.com/', + }, + done, + ); + }); - it('should redirect to https://google.com/about', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about`, - responseType: 'redirect', - redirectUrl: 'https://www.google.com/about', - }, done); - }); - }); + it('should redirect to https://google.com/about', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about`, + responseType: 'redirect', + redirectUrl: 'https://www.google.com/about', + }, + done, + ); + }); + }, + ); describe('with custom error document', () => { beforeEach(done => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, - Key: 'error.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html' }, done); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'error.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + }, + done, + ); }); }); @@ -321,55 +376,62 @@ describe('User visits bucket website endpoint', () => { s3.deleteObject({ Bucket: bucket, Key: 'error.html' }, done); }); - it('should serve custom error document if an error occurred', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'error-user', - }, done); - }); - - it('should serve custom error document with redirect', - done => { - s3.putObject({ Bucket: bucket, - Key: 'error.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://scality.com/test', - }, err => { - assert.ifError(err); - WebsiteConfigTester.checkHTML({ + it('should serve custom error document if an error occurred', done => { + WebsiteConfigTester.checkHTML( + { method: 'GET', url: endpoint, - responseType: 'redirect-error', - redirectUrl: 'https://scality.com/test', - expectedHeaders: { - 'x-amz-error-code': 'AccessDenied', - 'x-amz-error-message': 'Access Denied', - }, - }, done); - }); + responseType: 'error-user', + }, + done, + ); + }); + + it('should serve custom error document with redirect', done => { + s3.putObject( + { + Bucket: bucket, + Key: 'error.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://scality.com/test', + }, + err => { + assert.ifError(err); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect-error', + redirectUrl: 'https://scality.com/test', + expectedHeaders: { + 'x-amz-error-code': 'AccessDenied', + 'x-amz-error-message': 'Access Denied', + }, + }, + done, + ); + }, + ); }); }); describe('unfound custom error document', () => { beforeEach(done => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it('should serve s3 error file if unfound custom error document ' + - 'and an error occurred', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-retrieve-error-document', - }, done); + it('should serve s3 error file if unfound custom error document ' + 'and an error occurred', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-retrieve-error-document', + }, + done, + ); }); }); @@ -383,18 +445,19 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint} if error 403` + - ' occured', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/`, - }, done); + it(`should redirect to ${redirectEndpoint} if error 403` + ' occured', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/`, + }, + done, + ); }); }); @@ -408,23 +471,23 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint}/about/ if ` + - 'key prefix is equal to "about"', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); + it(`should redirect to ${redirectEndpoint}/about/ if ` + 'key prefix is equal to "about"', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); }); }); - describe.skip('redirect to hostname with prefix and error condition', - () => { + describe.skip('redirect to hostname with prefix and error condition', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -435,19 +498,23 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint} if ` + - 'key prefix is equal to "about" AND error code 403', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); - }); + it( + `should redirect to ${redirectEndpoint} if ` + 'key prefix is equal to "about" AND error code 403', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); + }, + ); }); describe.skip('redirect with multiple redirect rules', () => { @@ -464,22 +531,23 @@ describe('User visits bucket website endpoint', () => { }; webConfig.addRoutingRule(redirectOne, conditions); webConfig.addRoutingRule(redirectTwo, conditions); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); it('should redirect to the first one', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); }); }); - describe.skip('redirect with protocol', - () => { + describe.skip('redirect with protocol', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -490,18 +558,19 @@ describe('User visits bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it('should redirect to https://www.google.com/about if ' + - 'https protocols', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: 'https://www.google.com/about/', - }, done); + it('should redirect to https://www.google.com/about if ' + 'https protocols', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: 'https://www.google.com/about/', + }, + done, + ); }); }); @@ -513,23 +582,23 @@ describe('User visits bucket website endpoint', () => { const redirect = { ReplaceKeyWith: 'redirect.html', }; - putBucketWebsiteAndPutObjectRedirect(redirect, condition, - 'redirect.html', done); + putBucketWebsiteAndPutObjectRedirect(redirect, condition, 'redirect.html', done); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'redirect.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'redirect.html' }, err => done(err)); }); - it('should serve redirect file if error 403 error occured', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect.html`, - }, done); + it('should serve redirect file if error 403 error occured', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect.html`, + }, + done, + ); }); }); @@ -544,23 +613,23 @@ describe('User visits bucket website endpoint', () => { ReplaceKeyPrefixWith: 'about/', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, done); + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, done); }); - it(`should redirect to ${redirectEndpoint}/about/ if ` + - 'ReplaceKeyPrefixWith equals "about/"', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}/about/`, - }, done); + it(`should redirect to ${redirectEndpoint}/about/ if ` + 'ReplaceKeyPrefixWith equals "about/"', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}/about/`, + }, + done, + ); }); }); - describe.skip('redirect requests with prefix /about to redirect/', - () => { + describe.skip('redirect requests with prefix /about to redirect/', () => { beforeEach(done => { const condition = { KeyPrefixEquals: 'about/', @@ -568,184 +637,209 @@ describe('User visits bucket website endpoint', () => { const redirect = { ReplaceKeyPrefixWith: 'redirect/', }; - putBucketWebsiteAndPutObjectRedirect(redirect, condition, - 'redirect/index.html', done); + putBucketWebsiteAndPutObjectRedirect(redirect, condition, 'redirect/index.html', done); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, err => done(err)); }); - it('should serve redirect file if key prefix is equal to "about"', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect/`, - }, done); + it('should serve redirect file if key prefix is equal to "about"', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect/`, + }, + done, + ); }); }); - describe.skip('redirect requests, with prefix /about and that return ' + - '403 error, to prefix redirect/', () => { - beforeEach(done => { - const condition = { - KeyPrefixEquals: 'about/', - HttpErrorCodeReturnedEquals: '403', - }; - const redirect = { - ReplaceKeyPrefixWith: 'redirect/', - }; - putBucketWebsiteAndPutObjectRedirect(redirect, condition, - 'redirect/index.html', done); - }); + describe.skip( + 'redirect requests, with prefix /about and that return ' + '403 error, to prefix redirect/', + () => { + beforeEach(done => { + const condition = { + KeyPrefixEquals: 'about/', + HttpErrorCodeReturnedEquals: '403', + }; + const redirect = { + ReplaceKeyPrefixWith: 'redirect/', + }; + putBucketWebsiteAndPutObjectRedirect(redirect, condition, 'redirect/index.html', done); + }); - afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, - err => done(err)); - }); + afterEach(done => { + s3.deleteObject({ Bucket: bucket, Key: 'redirect/index.html' }, err => done(err)); + }); - it('should serve redirect file if key prefix is equal to ' + - '"about" and error 403', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect/`, - }, done); - }); - }); + it('should serve redirect file if key prefix is equal to ' + '"about" and error 403', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect/`, + }, + done, + ); + }); + }, + ); describe('object redirect to /', () => { beforeEach(done => { const webConfig = new WebsiteConfigTester('index.html'); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - Metadata: { - test: 'value', + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + Metadata: { + test: 'value', + }, + WebsiteRedirectLocation: '/', }, - WebsiteRedirectLocation: '/', - }, err => { assert.strictEqual(err, null); done(); - }); + }, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, err => done(err)); }); it('should redirect to /', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/index.html`, - responseType: 'redirect', - redirectUrl: '/', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/index.html`, + responseType: 'redirect', + redirectUrl: '/', + }, + done, + ); }); }); describe('with bucket policy', () => { beforeEach(done => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - - async.waterfall([ - next => s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, next), - (data, next) => s3.putBucketPolicy({ Bucket: bucket, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/index.html`, - `arn:aws:s3:::${bucket}/error.html`, - `arn:aws:s3:::${bucket}/access.html`, - ], - }, - { - Sid: 'DenyUnrelatedObj', - Effect: 'Deny', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/unrelated_obj.html`, - ], - }], - }), - }, next), - (data, next) => s3.putObject({ - Bucket: bucket, Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - }, next), - (data, next) => s3.putObject({ - Bucket: bucket, Key: 'error.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html', - }, next), - - ], err => { - assert.ifError(err); - done(); - }); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + + async.waterfall( + [ + next => s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, next), + (data, next) => + s3.putBucketPolicy( + { + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/index.html`, + `arn:aws:s3:::${bucket}/error.html`, + `arn:aws:s3:::${bucket}/access.html`, + ], + }, + { + Sid: 'DenyUnrelatedObj', + Effect: 'Deny', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [`arn:aws:s3:::${bucket}/unrelated_obj.html`], + }, + ], + }), + }, + next, + ), + (data, next) => + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, + next, + ), + (data, next) => + s3.putObject( + { + Bucket: bucket, + Key: 'error.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + }, + next, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); afterEach(done => { - async.waterfall([ - next => s3.deleteObject({ Bucket: bucket, - Key: 'index.html' }, next), - (data, next) => s3.deleteObject({ Bucket: bucket, - Key: 'error.html' }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, next), + (data, next) => s3.deleteObject({ Bucket: bucket, Key: 'error.html' }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should serve indexDocument if no key requested', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'index-user', + }, + done, + ); }); - it('should serve custom error 403 with deny on unrelated object ' + - 'and no access to key', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/non_existing.html`, - responseType: 'error-user', - }, done); + it('should serve custom error 403 with deny on unrelated object ' + 'and no access to key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/non_existing.html`, + responseType: 'error-user', + }, + done, + ); }); - it('should serve custom error 404 with deny on unrelated object ' + - 'and access to key', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/access.html`, - responseType: 'error-user-404', - }, done); + it('should serve custom error 404 with deny on unrelated object ' + 'and access to key', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/access.html`, + responseType: 'error-user-404', + }, + done, + ); }); }); @@ -759,34 +853,37 @@ describe('User visits bucket website endpoint', () => { ReplaceKeyWith: 'whatever.html', }; webConfig.addRoutingRule(redirect, condition); - s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, err => { - assert.strictEqual(err, - null, `Found unexpected err ${err}`); - s3.putObject({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - }, + s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + s3.putObject( + { + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }, err => { assert.strictEqual(err, null); done(); - }); + }, + ); }); }); afterEach(done => { - s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, - err => done(err)); + s3.deleteObject({ Bucket: bucket, Key: 'index.html' }, err => done(err)); }); it('should not redirect if index key is not explicit', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'index-user', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'index-user', + }, + done, + ); }); }); @@ -795,72 +892,84 @@ describe('User visits bucket website endpoint', () => { const webConfig = new WebsiteConfigTester('index.html'); const object = { Bucket: bucket, - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', }; - async.waterfall([ - next => s3.putBucketWebsite({ Bucket: bucket, - WebsiteConfiguration: webConfig }, next), - (data, next) => s3.putBucketPolicy({ Bucket: bucket, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/original_key_file`, - `arn:aws:s3:::${bucket}/original_key_nofile`, - `arn:aws:s3:::${bucket}/file/*`, - `arn:aws:s3:::${bucket}/nofile/*`, - ], - }], - }), - }, next), - (data, next) => s3.putObject(Object.assign({}, object, - { Key: 'original_key_file/index.html' }), next), - (data, next) => s3.putObject(Object.assign({}, object, - { Key: 'file/index.html' }), next), // the redirect 302 - (data, next) => s3.putObject(Object.assign({}, object, - { Key: 'no_access_file/index.html' }), next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => s3.putBucketWebsite({ Bucket: bucket, WebsiteConfiguration: webConfig }, next), + (data, next) => + s3.putBucketPolicy( + { + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/original_key_file`, + `arn:aws:s3:::${bucket}/original_key_nofile`, + `arn:aws:s3:::${bucket}/file/*`, + `arn:aws:s3:::${bucket}/nofile/*`, + ], + }, + ], + }), + }, + next, + ), + (data, next) => + s3.putObject(Object.assign({}, object, { Key: 'original_key_file/index.html' }), next), + (data, next) => s3.putObject(Object.assign({}, object, { Key: 'file/index.html' }), next), // the redirect 302 + (data, next) => + s3.putObject(Object.assign({}, object, { Key: 'no_access_file/index.html' }), next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); afterEach(done => { - async.waterfall([ - next => s3.deleteObject({ Bucket: bucket, - Key: 'original_key_file/index.html' }, next), - (data, next) => s3.deleteObject({ Bucket: bucket, - Key: 'file/index.html' }, next), - (data, next) => s3.deleteObject({ Bucket: bucket, - Key: 'no_access_file/index.html' }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => s3.deleteObject({ Bucket: bucket, Key: 'original_key_file/index.html' }, next), + (data, next) => s3.deleteObject({ Bucket: bucket, Key: 'file/index.html' }, next), + (data, next) => s3.deleteObject({ Bucket: bucket, Key: 'no_access_file/index.html' }, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should redirect 302 with trailing / on folder with index', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/file`, - responseType: 'redirect-error-found', - redirectUrl: '/file/', - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/file`, + responseType: 'redirect-error-found', + redirectUrl: '/file/', + }, + done, + ); }); - it('should return 404 on original key access without index', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/original_key_nofile`, - responseType: '404-not-found', - }, done); + it('should return 404 on original key access without index', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/original_key_nofile`, + responseType: '404-not-found', + }, + done, + ); }); describe('should return 403', () => { @@ -879,12 +988,16 @@ describe('User visits bucket website endpoint', () => { }, ].forEach(test => it(test.it, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/${test.key}`, - responseType: '403-access-denied', - }, done); - })); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/${test.key}`, + responseType: '403-access-denied', + }, + done, + ); + }), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js b/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js index 7bc84b7fd2..cb63aaafc6 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js +++ b/tests/functional/aws-node-sdk/test/object/websiteGetWithACL.js @@ -12,13 +12,11 @@ const s3 = new S3Client(config); // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; const aclEquivalent = { public: ['public-read-write', 'public-read'], @@ -33,87 +31,75 @@ const aclTests = [ html: '403-access-denied', }, { - it: 'should return 403 if public bucket - private index - public ' + - 'error documents', + it: 'should return 403 if public bucket - private index - public ' + 'error documents', bucketACL: 'public', objects: { index: 'private', error: 'private' }, html: '403-access-denied', }, { - it: 'should return index doc if private bucket - public index - ' + - 'public error documents', + it: 'should return index doc if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'private' }, html: 'index-user', }, { - it: 'should return index doc if public bucket - public index - ' + - 'private error documents', + it: 'should return index doc if public bucket - public index - ' + 'private error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'private' }, html: 'index-user', }, { - it: 'should return index doc if private bucket - public index - ' + - 'public error documents', + it: 'should return index doc if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'public-read' }, html: 'index-user', }, { - it: 'should return index doc if public bucket - public index - ' + - 'public error documents', + it: 'should return index doc if public bucket - public index - ' + 'public error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'public-read' }, html: 'index-user', }, { - it: 'should return error doc if private bucket - without index - ' + - 'public error documents', + it: 'should return error doc if private bucket - without index - ' + 'public error documents', bucketACL: 'private', objects: { error: 'public-read' }, html: 'error-user', }, { - it: 'should return 404 if public bucket - without index - ' + - 'public error documents', + it: 'should return 404 if public bucket - without index - ' + 'public error documents', bucketACL: 'public', objects: { error: 'public-read' }, html: 'error-user-404', }, { - it: 'should return 403 if private bucket - without index - ' + - 'private error documents', + it: 'should return 403 if private bucket - without index - ' + 'private error documents', bucketACL: 'private', objects: { error: 'private' }, html: '403-access-denied', }, { - it: 'should return 404 if public bucket - without index - ' + - 'private error documents', + it: 'should return 404 if public bucket - without index - ' + 'private error documents', bucketACL: 'public', objects: { error: 'private' }, html: '404-not-found', }, { - it: 'should return 404 if public bucket - without index - ' + - 'without error documents', + it: 'should return 404 if public bucket - without index - ' + 'without error documents', bucketACL: 'public', - objects: { }, + objects: {}, html: '404-not-found', }, { - it: 'should return 403 if private bucket - without index - ' + - 'without error documents', + it: 'should return 403 if private bucket - without index - ' + 'without error documents', bucketACL: 'private', - objects: { }, + objects: {}, html: '403-access-denied', }, - ]; describe('User visits bucket website endpoint with ACL', () => { @@ -121,12 +107,10 @@ describe('User visits bucket website endpoint with ACL', () => { aclEquivalent[test.bucketACL].forEach(bucketACL => { describe(`with existing bucket with ${bucketACL} acl`, () => { beforeEach(done => { - WebsiteConfigTester.createPutBucketWebsite(s3, bucket, - bucketACL, test.objects, done); + WebsiteConfigTester.createPutBucketWebsite(s3, bucket, bucketACL, test.objects, done); }); afterEach(done => { - WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, - test.objects, err => { + WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, test.objects, err => { if (process.env.AWS_ON_AIR) { // Give some time for AWS to finish deleting // object and buckets before starting next test @@ -138,29 +122,38 @@ describe('User visits bucket website endpoint with ACL', () => { }); it(`${test.it} with no auth credentials sent`, done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - requestType: test.html, - }, done); + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + requestType: test.html, + }, + done, + ); }); it(`${test.it} even with invalid auth credentials`, done => { - WebsiteConfigTester.checkHTML({ - auth: 'invalid credentials', - method: 'GET', - url: endpoint, - requestType: test.html, - }, done); + WebsiteConfigTester.checkHTML( + { + auth: 'invalid credentials', + method: 'GET', + url: endpoint, + requestType: test.html, + }, + done, + ); }); it(`${test.it} even with valid auth credentials`, done => { - WebsiteConfigTester.checkHTML({ - auth: 'valid credentials', - method: 'GET', - url: endpoint, - requestType: test.html, - }, done); + WebsiteConfigTester.checkHTML( + { + auth: 'valid credentials', + method: 'GET', + url: endpoint, + requestType: test.html, + }, + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteHead.js b/tests/functional/aws-node-sdk/test/object/websiteHead.js index 0d26fda09a..bbdccf4cde 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteHead.js +++ b/tests/functional/aws-node-sdk/test/object/websiteHead.js @@ -23,19 +23,16 @@ const s3 = new S3Client(config); // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? `awsbucketwebsitetester-${Date.now()}` : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; -const redirectEndpoint = conf.https ? 'https://www.google.com/' : - 'http://www.google.com/'; +const bucket = process.env.AWS_ON_AIR ? `awsbucketwebsitetester-${Date.now()}` : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; +const redirectEndpoint = conf.https ? 'https://www.google.com/' : 'http://www.google.com/'; const indexDocETag = '"95a589c37a2df74b062fb4d5a6f64197"'; const indexExpectedHeaders = { - 'etag': indexDocETag, + etag: indexDocETag, 'x-amz-meta-test': 'value', }; @@ -82,7 +79,6 @@ const indexExpectedHeaders = { // KX/MgqE4dZCJ4d9eF59Wbg/kza40cWcoA= // x-amz-request-id: 0073330F58C7137C - describe('Head request on bucket website endpoint', () => { it('should return 404 when no such bucket', done => { const expectedHeaders = { @@ -91,8 +87,7 @@ describe('Head request on bucket website endpoint', () => { // so compatible with aws 'x-amz-error-message': 'The specified bucket does not exist.', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, expectedHeaders, done); }); describe('with existing bucket', () => { @@ -103,95 +98,116 @@ describe('Head request on bucket website endpoint', () => { it('should return 404 when no website configuration', done => { const expectedHeaders = { 'x-amz-error-code': 'NoSuchWebsiteConfiguration', - 'x-amz-error-message': 'The specified bucket does not ' + - 'have a website configuration', + 'x-amz-error-message': 'The specified bucket does not ' + 'have a website configuration', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 404, expectedHeaders, done); }); describe('with existing configuration', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', Metadata: { test: 'value', }, - })); + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); - it('should return indexDocument headers if no key ' + - 'requested', done => { - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 200, indexExpectedHeaders, done); + it('should return indexDocument headers if no key ' + 'requested', done => { + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 200, indexExpectedHeaders, done); }); it('should return indexDocument headers if key requested', done => { - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/index.html`, 200, indexExpectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/index.html`, + 200, + indexExpectedHeaders, + done, + ); }); }); describe('with path prefix in request with/without key', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'pathprefix/index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - Metadata: { - test: 'value', - }, - })).catch(err => { + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); + await s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'pathprefix/index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + Metadata: { + test: 'value', + }, + }), + ) + .catch(err => { assert.strictEqual(err, null); }); }); afterEach(async () => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'pathprefix/index.html' }))); - it('should serve indexDocument if path request without key', - done => { - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/pathprefix/`, 200, indexExpectedHeaders, done); - }); - - it('should serve indexDocument if path request with key', - done => { - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/pathprefix/index.html`, 200, - indexExpectedHeaders, done); + it('should serve indexDocument if path request without key', done => { + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/pathprefix/`, + 200, + indexExpectedHeaders, + done, + ); + }); + + it('should serve indexDocument if path request with key', done => { + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/pathprefix/index.html`, + 200, + indexExpectedHeaders, + done, + ); }); }); describe('with private key', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'private', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html' })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); + await s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'private', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + }), + ) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); @@ -201,16 +217,14 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, expectedHeaders, done); }); }); describe('with nonexisting index document key', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); }); it('should return 403 if nonexisting index document key', done => { @@ -218,8 +232,7 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 403, expectedHeaders, done); }); }); @@ -228,94 +241,95 @@ describe('Head request on bucket website endpoint', () => { const redirectAllTo = { HostName: 'www.google.com', }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); it(`should redirect to ${redirectEndpoint}`, done => { const expectedHeaders = { location: redirectEndpoint, }; - WebsiteConfigTester.makeHeadRequest(undefined, - endpoint, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); it(`should redirect to ${redirectEndpoint}about`, done => { const expectedHeaders = { location: `${redirectEndpoint}about/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect all requests to https://www.google.com ' + - 'since https protocol set in website config', () => { - // Note: these tests will all redirect to https even if - // conf does not have https since protocol in website config - // specifies https - beforeEach(async () => { - const redirectAllTo = { - HostName: 'www.google.com', - Protocol: 'https', - }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); + describe( + 'redirect all requests to https://www.google.com ' + 'since https protocol set in website config', + () => { + // Note: these tests will all redirect to https even if + // conf does not have https since protocol in website config + // specifies https + beforeEach(async () => { + const redirectAllTo = { + HostName: 'www.google.com', + Protocol: 'https', + }; + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - }); - it('should redirect to https://google.com', done => { - const expectedHeaders = { - location: 'https://www.google.com/', - }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 301, expectedHeaders, done); - }); + it('should redirect to https://google.com', done => { + const expectedHeaders = { + location: 'https://www.google.com/', + }; + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); + }); - it('should redirect to https://google.com/about', done => { - const expectedHeaders = { - location: 'https://www.google.com/about/', - }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); - }); - }); + it('should redirect to https://google.com/about', done => { + const expectedHeaders = { + location: 'https://www.google.com/about/', + }; + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); + }); + }, + ); describe('with custom error document', () => { beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html', - 'error.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'error.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/error.html')), - ContentType: 'text/html' })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + const webConfig = new WebsiteConfigTester('index.html', 'error.html'); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); + await s3 + .send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'error.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/error.html')), + ContentType: 'text/html', + }), + ) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'error.html' }))); - it('should return regular error headers regardless of whether ' + - 'custom error document', done => { + it('should return regular error headers regardless of whether ' + 'custom error document', done => { const expectedHeaders = { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/madeup`, 403, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/madeup`, 403, expectedHeaders, done); }); }); @@ -329,19 +343,18 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint} if error 403` + - ' occured', done => { + it(`should redirect to ${redirectEndpoint} if error 403` + ' occured', done => { const expectedHeaders = { location: redirectEndpoint, }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); }); @@ -355,24 +368,22 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint}about if ` + - 'key prefix is equal to "about"', done => { + it(`should redirect to ${redirectEndpoint}about if ` + 'key prefix is equal to "about"', done => { const expectedHeaders = { location: `${redirectEndpoint}about/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect to hostname with prefix and error condition', - () => { + describe('redirect to hostname with prefix and error condition', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -383,20 +394,22 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint} if ` + - 'key prefix is equal to "about" AND error code 403', done => { - const expectedHeaders = { - location: `${redirectEndpoint}about/`, - }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); - }); + it( + `should redirect to ${redirectEndpoint} if ` + 'key prefix is equal to "about" AND error code 403', + done => { + const expectedHeaders = { + location: `${redirectEndpoint}about/`, + }; + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); + }, + ); }); describe('redirect with multiple redirect rules', () => { @@ -413,23 +426,22 @@ describe('Head request on bucket website endpoint', () => { }; webConfig.addRoutingRule(redirectOne, conditions); webConfig.addRoutingRule(redirectTwo, conditions); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); it('should redirect based on first rule', done => { const expectedHeaders = { location: `${redirectEndpoint}about/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect with protocol', - () => { + describe('redirect with protocol', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -440,19 +452,18 @@ describe('Head request on bucket website endpoint', () => { HostName: 'www.google.com', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it('should redirect to https://www.google.com/about if ' + - 'https protocol specified', done => { + it('should redirect to https://www.google.com/about if ' + 'https protocol specified', done => { const expectedHeaders = { location: 'https://www.google.com/about/', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); @@ -466,21 +477,20 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyWith: 'redirect.html', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'redirect.html' }))); - it('should redirect to specified file if 403 error ' + - 'error occured', done => { + it('should redirect to specified file if 403 error ' + 'error occured', done => { const expectedHeaders = { location: `${endpoint}/redirect.html`, }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); }); @@ -495,24 +505,22 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyPrefixWith: 'about', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); - it(`should redirect to ${redirectEndpoint}about if ` + - 'ReplaceKeyPrefixWith equals "about"', done => { + it(`should redirect to ${redirectEndpoint}about if ` + 'ReplaceKeyPrefixWith equals "about"', done => { const expectedHeaders = { location: `${redirectEndpoint}about`, }; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 301, expectedHeaders, done); }); }); - describe('redirect requests with prefix /about to redirect/', - () => { + describe('redirect requests with prefix /about to redirect/', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -522,27 +530,24 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyPrefixWith: 'redirect/', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(async () => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'redirect/index.html' }))); - - it('should redirect to "redirect/" object if key prefix is equal ' + - 'to "about/"', done => { + it('should redirect to "redirect/" object if key prefix is equal ' + 'to "about/"', done => { const expectedHeaders = { location: `${endpoint}/redirect/`, }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); }); }); - describe('redirect requests, with both prefix and error code ' + - 'condition', () => { + describe('redirect requests, with both prefix and error code ' + 'condition', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); const condition = { @@ -553,41 +558,45 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyPrefixWith: 'redirect/', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })).catch(err => { - assert.strictEqual(err, null, `Found unexpected err ${err}`); - }); + await s3 + .send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })) + .catch(err => { + assert.strictEqual(err, null, `Found unexpected err ${err}`); + }); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'redirect/index.html' }))); - it('should redirect to "redirect" object if key prefix is equal ' + - 'to "about/" and there is a 403 error satisfying the ' + - 'condition in the redirect rule', - done => { - const expectedHeaders = { - location: `${endpoint}/redirect/`, - }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/about/`, 301, expectedHeaders, done); - }); + it( + 'should redirect to "redirect" object if key prefix is equal ' + + 'to "about/" and there is a 403 error satisfying the ' + + 'condition in the redirect rule', + done => { + const expectedHeaders = { + location: `${endpoint}/redirect/`, + }; + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/about/`, 301, expectedHeaders, done); + }, + ); }); describe('object redirect to /', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', Metadata: { test: 'value', }, WebsiteRedirectLocation: '/', - })); + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); @@ -596,46 +605,51 @@ describe('Head request on bucket website endpoint', () => { const expectedHeaders = { location: '/', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/index.html`, 301, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/index.html`, 301, expectedHeaders, done); }); }); describe('with bucket policy', () => { beforeEach(async () => { const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutBucketPolicyCommand({ Bucket: bucket, Policy: JSON.stringify( - { - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/index.html`, - `arn:aws:s3:::${bucket}/access.html`, + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutBucketPolicyCommand({ + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/index.html`, + `arn:aws:s3:::${bucket}/access.html`, + ], + }, ], - }], - } - )})); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + }), + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', Metadata: { test: 'value', - }})); + }, + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); - it('should return indexDocument headers if no key ' + - 'requested', done => { - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 200, indexExpectedHeaders, done); + it('should return indexDocument headers if no key ' + 'requested', done => { + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 200, indexExpectedHeaders, done); }); it('should serve error 403 with no access to key', done => { @@ -643,9 +657,13 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/non_existing.html`, 403, expectedHeaders, - done); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/non_existing.html`, + 403, + expectedHeaders, + done, + ); }); it('should serve error 404 with access to key', done => { @@ -653,9 +671,7 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'NoSuchKey', 'x-amz-error-message': 'The specified key does not exist.', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/access.html`, 404, expectedHeaders, - done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/access.html`, 404, expectedHeaders, done); }); }); @@ -669,24 +685,25 @@ describe('Head request on bucket website endpoint', () => { ReplaceKeyWith: 'whatever.html', }; webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - Metadata: { - test: 'value', - }, - })); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + Metadata: { + test: 'value', + }, + }), + ); }); afterEach(() => s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'index.html' }))); it('should not redirect if index key is not explicit', done => { - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, - 200, indexExpectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, endpoint, 200, indexExpectedHeaders, done); }); }); @@ -695,66 +712,65 @@ describe('Head request on bucket website endpoint', () => { const webConfig = new WebsiteConfigTester('index.html'); const object = { Bucket: bucket, - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), ContentType: 'text/html', }; - - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutBucketPolicyCommand({ Bucket: bucket, - Policy: JSON.stringify({ - Version: '2012-10-17', - Statement: [{ - Sid: 'PublicReadGetObject', - Effect: 'Allow', - Principal: '*', - Action: ['s3:GetObject'], - Resource: [ - `arn:aws:s3:::${bucket}/original_key_file`, - `arn:aws:s3:::${bucket}/original_key_nofile`, - `arn:aws:s3:::${bucket}/file/*`, - `arn:aws:s3:::${bucket}/nofile/*`, + + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutBucketPolicyCommand({ + Bucket: bucket, + Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [ + { + Sid: 'PublicReadGetObject', + Effect: 'Allow', + Principal: '*', + Action: ['s3:GetObject'], + Resource: [ + `arn:aws:s3:::${bucket}/original_key_file`, + `arn:aws:s3:::${bucket}/original_key_nofile`, + `arn:aws:s3:::${bucket}/file/*`, + `arn:aws:s3:::${bucket}/nofile/*`, + ], + }, ], - }], - }) - })); - await s3.send(new PutObjectCommand(Object.assign({}, object, - { Key: 'original_key_file/index.html' }))); - await s3.send(new PutObjectCommand(Object.assign({}, object, - { Key: 'file/index.html' }))); - await s3.send(new PutObjectCommand(Object.assign({}, object, - { Key: 'no_access_file/index.html' }))); + }), + }), + ); + await s3.send(new PutObjectCommand(Object.assign({}, object, { Key: 'original_key_file/index.html' }))); + await s3.send(new PutObjectCommand(Object.assign({}, object, { Key: 'file/index.html' }))); + await s3.send(new PutObjectCommand(Object.assign({}, object, { Key: 'no_access_file/index.html' }))); }); afterEach(async () => { - await s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: 'original_key_file/index.html' })); - await s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: 'file/index.html' })); - await s3.send(new DeleteObjectCommand({ Bucket: bucket, - Key: 'no_access_file/index.html' })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'original_key_file/index.html' })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'file/index.html' })); + await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: 'no_access_file/index.html' })); }); it('should redirect 302 with trailing / on folder with index', done => { const expectedHeaders = { - 'location': '/file/', + location: '/file/', 'x-amz-error-code': 'Found', 'x-amz-error-message': 'Resource Found', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/file`, 302, expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest(undefined, `${endpoint}/file`, 302, expectedHeaders, done); }); - it('should return 404 on original key access without index', - done => { + it('should return 404 on original key access without index', done => { const expectedHeaders = { 'x-amz-error-code': 'NoSuchKey', 'x-amz-error-message': 'The specified key does not exist.', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/original_key_nofile`, 404, - expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/original_key_nofile`, + 404, + expectedHeaders, + done, + ); }); describe('should return 403', () => { @@ -777,10 +793,15 @@ describe('Head request on bucket website endpoint', () => { 'x-amz-error-code': 'AccessDenied', 'x-amz-error-message': 'Access Denied', }; - WebsiteConfigTester.makeHeadRequest(undefined, - `${endpoint}/${test.key}`, 403, - expectedHeaders, done); - })); + WebsiteConfigTester.makeHeadRequest( + undefined, + `${endpoint}/${test.key}`, + 403, + expectedHeaders, + done, + ); + }), + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js b/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js index 79ecf2d2b5..1ed267396f 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js +++ b/tests/functional/aws-node-sdk/test/object/websiteHeadWithACL.js @@ -12,13 +12,11 @@ const s3 = new S3Client(config); // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; const aclEquivalent = { public: ['public-read-write', 'public-read'], @@ -56,84 +54,73 @@ const aclTests = [ result: 'accessDenied', }, { - it: 'should return 403 if public bucket - private index - public ' + - 'error documents', + it: 'should return 403 if public bucket - private index - public ' + 'error documents', bucketACL: 'public', objects: { index: 'private', error: 'private' }, result: 'accessDenied', }, { - it: 'should return 200 if private bucket - public index - ' + - 'public error documents', + it: 'should return 200 if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'private' }, result: 'index', }, { - it: 'should return 200 if public bucket - public index - ' + - 'private error documents', + it: 'should return 200 if public bucket - public index - ' + 'private error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'private' }, result: 'index', }, { - it: 'should return 200 if private bucket - public index - ' + - 'public error documents', + it: 'should return 200 if private bucket - public index - ' + 'public error documents', bucketACL: 'private', objects: { index: 'public-read', error: 'public-read' }, result: 'index', }, { - it: 'should return 200 if public bucket - public index - ' + - 'public error documents', + it: 'should return 200 if public bucket - public index - ' + 'public error documents', bucketACL: 'public', objects: { index: 'public-read', error: 'public-read' }, result: 'index', }, { - it: 'should return 403 AccessDenied if private bucket - ' + - 'without index - public error documents', + it: 'should return 403 AccessDenied if private bucket - ' + 'without index - public error documents', bucketACL: 'private', objects: { error: 'public-read' }, result: 'accessDenied', }, { - it: 'should return 404 if public bucket - without index - ' + - 'public error documents', + it: 'should return 404 if public bucket - without index - ' + 'public error documents', bucketACL: 'public', objects: { error: 'public-read' }, result: 'noSuchKey', }, { - it: 'should return 403 if private bucket - without index - ' + - 'private error documents', + it: 'should return 403 if private bucket - without index - ' + 'private error documents', bucketACL: 'private', objects: { error: 'private' }, result: 'accessDenied', }, { - it: 'should return 404 if public bucket - without index - ' + - 'private error documents', + it: 'should return 404 if public bucket - without index - ' + 'private error documents', bucketACL: 'public', objects: { error: 'private' }, result: 'noSuchKey', }, { - it: 'should return 404 if public bucket - without index - ' + - 'without error documents', + it: 'should return 404 if public bucket - without index - ' + 'without error documents', bucketACL: 'public', - objects: { }, + objects: {}, result: 'noSuchKey', }, { - it: 'should return 403 if private bucket - without index - ' + - 'without error documents', + it: 'should return 403 if private bucket - without index - ' + 'without error documents', bucketACL: 'private', - objects: { }, + objects: {}, result: 'accessDenied', }, ]; @@ -143,33 +130,43 @@ describe('Head request on bucket website endpoint with ACL', () => { aclEquivalent[test.bucketACL].forEach(bucketACL => { describe(`with existing bucket with ${bucketACL} acl`, () => { beforeEach(done => { - WebsiteConfigTester.createPutBucketWebsite(s3, bucket, - bucketACL, test.objects, done); + WebsiteConfigTester.createPutBucketWebsite(s3, bucket, bucketACL, test.objects, done); }); afterEach(done => { - WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, - test.objects, done); + WebsiteConfigTester.deleteObjectsThenBucket(s3, bucket, test.objects, done); }); it(`${test.it} with no auth credentials sent`, done => { const result = test.result; - WebsiteConfigTester.makeHeadRequest(undefined, endpoint, + WebsiteConfigTester.makeHeadRequest( + undefined, + endpoint, headersACL[result].status, - headersACL[result].expectedHeaders, done); + headersACL[result].expectedHeaders, + done, + ); }); it(`${test.it} even with invalid auth credentials`, done => { const result = test.result; - WebsiteConfigTester.makeHeadRequest('invalid credentials', - endpoint, headersACL[result].status, - headersACL[result].expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + 'invalid credentials', + endpoint, + headersACL[result].status, + headersACL[result].expectedHeaders, + done, + ); }); it(`${test.it} even with valid auth credentials`, done => { const result = test.result; - WebsiteConfigTester.makeHeadRequest('valid credentials', - endpoint, headersACL[result].status, - headersACL[result].expectedHeaders, done); + WebsiteConfigTester.makeHeadRequest( + 'valid credentials', + endpoint, + headersACL[result].status, + headersACL[result].expectedHeaders, + done, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js b/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js index 71acdda127..5e218c13e7 100644 --- a/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js +++ b/tests/functional/aws-node-sdk/test/object/websiteRuleMixing.js @@ -21,326 +21,405 @@ const s3 = bucketUtil.s3; // `127.0.0.1 bucketwebsitetester.s3-website-us-east-1.amazonaws.com` const transport = conf.https ? 'https' : 'http'; -const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : - 'bucketwebsitetester'; -const hostname = process.env.S3_END_TO_END ? - `${bucket}.s3-website-us-east-1.scality.com` : - `${bucket}.s3-website-us-east-1.amazonaws.com`; -const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : - `${transport}://${hostname}:8000`; -const redirectEndpoint = conf.https ? 'https://www.google.com/' : - 'http://www.google.com/'; - -describe('User visits bucket website endpoint and requests resource ' + -'that has x-amz-website-redirect-location header ::', () => { - beforeEach(async () => await s3.send(new CreateBucketCommand({ Bucket: bucket }))); - - afterEach(async () => await s3.send(new DeleteBucketCommand({ Bucket: bucket }))); - - describe('when x-amz-website-redirect-location: /redirect.html', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'redirect.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' })); - }); - - afterEach(async () => await bucketUtil.empty(bucket)); - - it('should serve redirect file on GET request', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: '/redirect.html', - }, done); - }); - - it('should redirect to redirect file on HEAD request', done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: '/redirect.html', - }, done); - }); - }); - - describe('when x-amz-website-redirect-location: https://www.google.com', +const bucket = process.env.AWS_ON_AIR ? 'awsbucketwebsitetester' : 'bucketwebsitetester'; +const hostname = process.env.S3_END_TO_END + ? `${bucket}.s3-website-us-east-1.scality.com` + : `${bucket}.s3-website-us-east-1.amazonaws.com`; +const endpoint = process.env.AWS_ON_AIR ? `${transport}://${hostname}` : `${transport}://${hostname}:8000`; +const redirectEndpoint = conf.https ? 'https://www.google.com/' : 'http://www.google.com/'; + +describe( + 'User visits bucket website endpoint and requests resource ' + 'that has x-amz-website-redirect-location header ::', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://www.google.com' })); - }); + beforeEach(async () => await s3.send(new CreateBucketCommand({ Bucket: bucket }))); + + afterEach(async () => await s3.send(new DeleteBucketCommand({ Bucket: bucket }))); + + describe('when x-amz-website-redirect-location: /redirect.html', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'redirect.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); + afterEach(async () => await bucketUtil.empty(bucket)); + + it('should serve redirect file on GET request', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: '/redirect.html', + }, + done, + ); + }); - it('should redirect to https://www.google.com', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: 'https://www.google.com', - }, done); + it('should redirect to redirect file on HEAD request', done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: '/redirect.html', + }, + done, + ); + }); }); - it('should redirect to https://www.google.com on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: 'https://www.google.com', - }, done); + describe('when x-amz-website-redirect-location: https://www.google.com', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://www.google.com', + }), + ); }); - }); - - describe('when key with header is private', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://www.google.com' })); - }); - afterEach(async () => await bucketUtil.empty(bucket)); + afterEach(async () => await bucketUtil.empty(bucket)); + + it('should redirect to https://www.google.com', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: 'https://www.google.com', + }, + done, + ); + }); - it('should return 403 instead of x-amz-website-redirect-location ' + - 'header location', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: '403-access-denied', - }, done); + it('should redirect to https://www.google.com on HEAD request', done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: 'https://www.google.com', + }, + done, + ); + }); }); - it('should return 403 instead of x-amz-website-redirect-location ' + - 'header location on HEAD request', done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: '403-access-denied', - }, done); - }); - }); - - describe('when key with header is private' + - 'and website config has error condition routing rule', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - const condition = { - HttpErrorCodeReturnedEquals: '403', - }; - const redirect = { - HostName: 'www.google.com', - }; - webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'redirect.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' })); - }); + describe('when key with header is private', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://www.google.com', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it(`should redirect to ${redirectEndpoint} since error 403 ` + - 'occurred instead of x-amz-website-redirect-location header ' + - 'location on GET request', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); - }); + afterEach(async () => await bucketUtil.empty(bucket)); + + it('should return 403 instead of x-amz-website-redirect-location ' + 'header location', done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); + }); - it(`should redirect to ${redirectEndpoint} since error 403 ` + - 'occurred instead of x-amz-website-redirect-location header ' + - 'location on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); - }); - }); - - describe(`with redirect all requests to ${redirectEndpoint}`, () => { - beforeEach(async () => { - const redirectAllTo = { - HostName: 'www.google.com', - }; - const webConfig = new WebsiteConfigTester(null, null, - redirectAllTo); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); + it( + 'should return 403 instead of x-amz-website-redirect-location ' + 'header location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: '403-access-denied', + }, + done, + ); + }, + ); }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it(`should redirect to ${redirectEndpoint} instead of ` + - 'x-amz-website-redirect-location header location on GET request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); - }); + describe('when key with header is private' + 'and website config has error condition routing rule', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + const condition = { + HttpErrorCodeReturnedEquals: '403', + }; + const redirect = { + HostName: 'www.google.com', + }; + webConfig.addRoutingRule(redirect, condition); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'redirect.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }), + ); + }); - it(`should redirect to ${redirectEndpoint} instead of ` + - 'x-amz-website-redirect-location header location on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: endpoint, - responseType: 'redirect', - redirectUrl: redirectEndpoint, - }, done); + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + `should redirect to ${redirectEndpoint} since error 403 ` + + 'occurred instead of x-amz-website-redirect-location header ' + + 'location on GET request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); + + it( + `should redirect to ${redirectEndpoint} since error 403 ` + + 'occurred instead of x-amz-website-redirect-location header ' + + 'location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); }); - }); - describe('with routing rule redirect to hostname with prefix condition', - () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - const condition = { - KeyPrefixEquals: 'about/', - }; - const redirect = { - HostName: 'www.google.com', - }; - webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'about/index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: '/redirect.html' })); - }); + describe(`with redirect all requests to ${redirectEndpoint}`, () => { + beforeEach(async () => { + const redirectAllTo = { + HostName: 'www.google.com', + }; + const webConfig = new WebsiteConfigTester(null, null, redirectAllTo); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it(`should redirect GET request to ${redirectEndpoint}about/ ` + - 'instead of about/ key x-amz-website-redirect-location ' + - 'header location', done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}about/`, - }, done); + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + `should redirect to ${redirectEndpoint} instead of ` + + 'x-amz-website-redirect-location header location on GET request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); + + it( + `should redirect to ${redirectEndpoint} instead of ` + + 'x-amz-website-redirect-location header location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: endpoint, + responseType: 'redirect', + redirectUrl: redirectEndpoint, + }, + done, + ); + }, + ); }); - it(`should redirect HEAD request to ${redirectEndpoint}about ` + - 'instead of about/ key x-amz-website-redirect-location ' + - 'header location', done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: `${endpoint}/about/`, - responseType: 'redirect', - redirectUrl: `${redirectEndpoint}about/`, - }, done); - }); - }); - - describe('with routing rule replaceKeyWith', () => { - beforeEach(async () => { - const webConfig = new WebsiteConfigTester('index.html'); - const condition = { - KeyPrefixEquals: 'index.html', - }; - const redirect = { - ReplaceKeyWith: 'redirect.html', - }; - webConfig.addRoutingRule(redirect, condition); - await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, - WebsiteConfiguration: webConfig })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'index.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/index.html')), - ContentType: 'text/html', - WebsiteRedirectLocation: 'https://www.google.com' })); - await s3.send(new PutObjectCommand({ Bucket: bucket, - Key: 'redirect.html', - ACL: 'public-read', - Body: fs.readFileSync(path.join(__dirname, - '/websiteFiles/redirect.html')), - ContentType: 'text/html' })); - }); + describe('with routing rule redirect to hostname with prefix condition', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + const condition = { + KeyPrefixEquals: 'about/', + }; + const redirect = { + HostName: 'www.google.com', + }; + webConfig.addRoutingRule(redirect, condition); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'about/index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: '/redirect.html', + }), + ); + }); - afterEach(async () => await bucketUtil.empty(bucket)); - - it('should replace key instead of redirecting to key ' + - 'x-amz-website-redirect-location header location on GET request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'GET', - url: `${endpoint}/index.html`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect.html`, - }, done); + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + `should redirect GET request to ${redirectEndpoint}about/ ` + + 'instead of about/ key x-amz-website-redirect-location ' + + 'header location', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}about/`, + }, + done, + ); + }, + ); + + it( + `should redirect HEAD request to ${redirectEndpoint}about ` + + 'instead of about/ key x-amz-website-redirect-location ' + + 'header location', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: `${endpoint}/about/`, + responseType: 'redirect', + redirectUrl: `${redirectEndpoint}about/`, + }, + done, + ); + }, + ); }); - it('should replace key instead of redirecting to key ' + - 'x-amz-website-redirect-location header location on HEAD request', - done => { - WebsiteConfigTester.checkHTML({ - method: 'HEAD', - url: `${endpoint}/index.html`, - responseType: 'redirect-user', - redirectUrl: `${endpoint}/redirect.html`, - }, done); + describe('with routing rule replaceKeyWith', () => { + beforeEach(async () => { + const webConfig = new WebsiteConfigTester('index.html'); + const condition = { + KeyPrefixEquals: 'index.html', + }; + const redirect = { + ReplaceKeyWith: 'redirect.html', + }; + webConfig.addRoutingRule(redirect, condition); + await s3.send(new PutBucketWebsiteCommand({ Bucket: bucket, WebsiteConfiguration: webConfig })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'index.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/index.html')), + ContentType: 'text/html', + WebsiteRedirectLocation: 'https://www.google.com', + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'redirect.html', + ACL: 'public-read', + Body: fs.readFileSync(path.join(__dirname, '/websiteFiles/redirect.html')), + ContentType: 'text/html', + }), + ); }); - }); -}); + + afterEach(async () => await bucketUtil.empty(bucket)); + + it( + 'should replace key instead of redirecting to key ' + + 'x-amz-website-redirect-location header location on GET request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'GET', + url: `${endpoint}/index.html`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect.html`, + }, + done, + ); + }, + ); + + it( + 'should replace key instead of redirecting to key ' + + 'x-amz-website-redirect-location header location on HEAD request', + done => { + WebsiteConfigTester.checkHTML( + { + method: 'HEAD', + url: `${endpoint}/index.html`, + responseType: 'redirect-user', + redirectUrl: `${endpoint}/redirect.html`, + }, + done, + ); + }, + ); + }); + }, +); diff --git a/tests/functional/aws-node-sdk/test/quota/tooling.js b/tests/functional/aws-node-sdk/test/quota/tooling.js index e9f616459d..ef1519f81d 100644 --- a/tests/functional/aws-node-sdk/test/quota/tooling.js +++ b/tests/functional/aws-node-sdk/test/quota/tooling.js @@ -41,14 +41,14 @@ const sendRequest = async (method, host, path, body = '', config = null, signing // Get credentials - use same source as S3 client configuration let accessKeyId = config?.accessKey || config?.accessKeyId; let secretAccessKey = config?.secretKey || config?.secretAccessKey; - + // If not provided in config, use getCredentials (matches S3 client credential source) if (!accessKeyId || !secretAccessKey) { const defaultCreds = getCredentials('default'); accessKeyId = accessKeyId || defaultCreds.accessKeyId; secretAccessKey = secretAccessKey || defaultCreds.secretAccessKey; } - + if (!accessKeyId || !secretAccessKey) { throw new Error('Missing accessKeyId or secretAccessKey in config'); } diff --git a/tests/functional/aws-node-sdk/test/rateLimit/client.js b/tests/functional/aws-node-sdk/test/rateLimit/client.js index 083eb6271b..c455c71a42 100644 --- a/tests/functional/aws-node-sdk/test/rateLimit/client.js +++ b/tests/functional/aws-node-sdk/test/rateLimit/client.js @@ -30,7 +30,7 @@ skipIfRateLimitDisabled('RateLimitClient', () => { assert.strictEqual(client.isReady(), true); }); - it('should return true when the client is waiting to connect or the first time', () => { + it('should return true when the client is waiting to connect or the first time', () => { const client = new RateLimitClient(config.localCache); assert.strictEqual(client.isReady(), true); }); diff --git a/tests/functional/aws-node-sdk/test/service/get.js b/tests/functional/aws-node-sdk/test/service/get.js index 4352ab4056..18827567c1 100644 --- a/tests/functional/aws-node-sdk/test/service/get.js +++ b/tests/functional/aws-node-sdk/test/service/get.js @@ -1,29 +1,21 @@ const assert = require('assert'); const tv4 = require('tv4'); const async = require('async'); -const { - S3Client, - ListBucketsCommand, - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, ListBucketsCommand, CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const BucketUtility = require('../../lib/utility/bucket-util'); const getConfig = require('../support/config'); const withV4 = require('../support/withV4'); const svcSchema = require('../../schema/service'); -const describeFn = process.env.AWS_ON_AIR - ? describe.skip - : describe; +const describeFn = process.env.AWS_ON_AIR ? describe.skip : describe; async function cleanBucket(bucketUtils, s3, Bucket) { try { await bucketUtils.empty(Bucket, true); await bucketUtils.deleteOne(Bucket); } catch (error) { - process.stdout - .write(`Error emptying and deleting bucket: ${error}\n`); + process.stdout.write(`Error emptying and deleting bucket: ${error}\n`); // ignore the error and continue } } @@ -35,9 +27,9 @@ async function cleanAllBuckets(bucketUtils, s3) { const list = await s3.send(new ListBucketsCommand({})); if (list.Buckets && list.Buckets.length) { - process.stdout - .write(`Found ${list.Buckets.length} buckets to clean:\n${ - JSON.stringify(list.Buckets, null, 2)}\n`); + process.stdout.write( + `Found ${list.Buckets.length} buckets to clean:\n${JSON.stringify(list.Buckets, null, 2)}\n`, + ); // clean sequentially to avoid overloading for (const bucket of list.Buckets) { @@ -51,7 +43,6 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { let unauthenticatedBucketUtil; describe('When user is unauthorized', () => { - beforeEach(() => { const config = getConfig('default'); unauthenticatedBucketUtil = new BucketUtility('default', config, true); @@ -87,18 +78,19 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { }; }); - it('should return 403 and InvalidAccessKeyId ' + - 'if accessKeyId is invalid', async () => { - const invalidAccess = getConfig('default', - Object.assign({}, + it('should return 403 and InvalidAccessKeyId ' + 'if accessKeyId is invalid', async () => { + const invalidAccess = getConfig( + 'default', + Object.assign( + {}, { credentials: { accessKeyId: 'wrong', secretAccessKey: 'wrong again', }, }, - sigCfg - ) + sigCfg, + ), ); const expectedCode = 'InvalidAccessKeyId'; const expectedStatus = 403; @@ -106,8 +98,7 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { await testFn(invalidAccess, expectedCode, expectedStatus); }); - it('should return 403 and SignatureDoesNotMatch ' + - 'if credential is polluted', async () => { + it('should return 403 and SignatureDoesNotMatch ' + 'if credential is polluted', async () => { const pollutedConfig = getConfig('default', sigCfg); pollutedConfig.credentials.secretAccessKey = 'wrong'; @@ -122,82 +113,97 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { let bucketUtil; let s3; const bucketsNumber = 1001; - process.stdout - .write(`testing listing with ${bucketsNumber} buckets\n`); - const createdBuckets = Array.from(Array(bucketsNumber).keys()) - .map(i => `getservicebuckets-${i}`); + process.stdout.write(`testing listing with ${bucketsNumber} buckets\n`); + const createdBuckets = Array.from(Array(bucketsNumber).keys()).map(i => `getservicebuckets-${i}`); before(done => { bucketUtil = new BucketUtility('default', sigCfg); s3 = bucketUtil.s3; - async.series([ - next => cleanAllBuckets(bucketUtil, s3).then(() => next()).catch(next), - next => - async.eachLimit(createdBuckets, 10, (bucketName, moveOn) => { - s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .then(() => { - if (bucketName.endsWith('000')) { - process.stdout - .write(`creating bucket: ${bucketName}\n`); + async.series( + [ + next => + cleanAllBuckets(bucketUtil, s3) + .then(() => next()) + .catch(next), + next => + async.eachLimit( + createdBuckets, + 10, + (bucketName, moveOn) => { + s3.send(new CreateBucketCommand({ Bucket: bucketName })) + .then(() => { + if (bucketName.endsWith('000')) { + process.stdout.write(`creating bucket: ${bucketName}\n`); + } + moveOn(); + }) + .catch(err => { + moveOn(err); + }); + }, + err => { + if (err) { + process.stdout.write(`err creating buckets: ${err}\n`); + return next(err); } - moveOn(); - }) - .catch(err => { - moveOn(err); - }); - }, - err => { - if (err) { - process.stdout.write(`err creating buckets: ${err}\n`); - return next(err); - } - return next(err); - }) - ], done); + return next(err); + }, + ), + ], + done, + ); }); after(done => { - async.eachLimit(createdBuckets, 10, (bucketName, moveOn) => { - s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => { - if (bucketName.endsWith('000')) { - // log to keep ci alive - process.stdout - .write(`deleting bucket: ${bucketName}\n`); - } - moveOn(); - }) - .catch(() => { - moveOn(); - }); - }, - err => { - if (err) { - process.stdout.write(`err deleting buckets: ${err}`); - } - done(err); - }); + async.eachLimit( + createdBuckets, + 10, + (bucketName, moveOn) => { + s3.send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => { + if (bucketName.endsWith('000')) { + // log to keep ci alive + process.stdout.write(`deleting bucket: ${bucketName}\n`); + } + moveOn(); + }) + .catch(() => { + moveOn(); + }); + }, + err => { + if (err) { + process.stdout.write(`err deleting buckets: ${err}`); + } + done(err); + }, + ); }); it('should list buckets concurrently', done => { - async.times(20, (n, next) => { - s3.send(new ListBucketsCommand({})) - .then(result => { - // Filter for our test buckets only - const ourBuckets = result.Buckets.filter(bucket => - bucket.Name.startsWith('getservicebuckets-') - ); - assert.equal(ourBuckets.length, - createdBuckets.length, - 'Created buckets are missing in response'); - next(); - }) - .catch(next); - }, - err => { - assert.ifError(err, `error listing buckets: ${err}`); - done(); - }); + async.times( + 20, + (n, next) => { + s3.send(new ListBucketsCommand({})) + .then(result => { + // Filter for our test buckets only + const ourBuckets = result.Buckets.filter(bucket => + bucket.Name.startsWith('getservicebuckets-'), + ); + assert.equal( + ourBuckets.length, + createdBuckets.length, + 'Created buckets are missing in response', + ); + next(); + }) + .catch(next); + }, + err => { + assert.ifError(err, `error listing buckets: ${err}`); + done(); + }, + ); }); it('should list buckets', done => { @@ -212,12 +218,9 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { return data; }) .then(data => { - const buckets = data.Buckets.filter(bucket => - createdBuckets.indexOf(bucket.Name) > -1 - ); + const buckets = data.Buckets.filter(bucket => createdBuckets.indexOf(bucket.Name) > -1); - assert.equal(buckets.length, createdBuckets.length, - 'Created buckets are missing in response'); + assert.equal(buckets.length, createdBuckets.length, 'Created buckets are missing in response'); return buckets; }) @@ -225,14 +228,12 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { // Sort createdBuckets in alphabetical order createdBuckets.sort(); - const isCorrectOrder = buckets - .reduce( - (prev, bucket, idx) => - prev && bucket.Name === createdBuckets[idx] - , true); + const isCorrectOrder = buckets.reduce( + (prev, bucket, idx) => prev && bucket.Name === createdBuckets[idx], + true, + ); - assert.ok(isCorrectOrder, - 'Not returning created buckets by alphabetically'); + assert.ok(isCorrectOrder, 'Not returning created buckets by alphabetically'); done(); }) .catch(done); @@ -248,14 +249,12 @@ describeFn('GET Service - AWS.S3.listBuckets', function getService() { }); it('should not return other accounts bucket list', done => { - anotherS3.send(new ListBucketsCommand({})) + anotherS3 + .send(new ListBucketsCommand({})) .then(data => { - const hasSameBuckets = data.Buckets - .filter(filterFn) - .length; + const hasSameBuckets = data.Buckets.filter(filterFn).length; - assert.strictEqual(hasSameBuckets, 0, - 'It has other buddies bucket'); + assert.strictEqual(hasSameBuckets, 0, 'It has other buddies bucket'); done(); }) .catch(done); diff --git a/tests/functional/aws-node-sdk/test/support/awsConfig.js b/tests/functional/aws-node-sdk/test/support/awsConfig.js index d19ab338c0..d6e02734e6 100644 --- a/tests/functional/aws-node-sdk/test/support/awsConfig.js +++ b/tests/functional/aws-node-sdk/test/support/awsConfig.js @@ -17,14 +17,14 @@ function getAwsCredentials(profile, credFile = '/.aws/credentials') { // Parse the INI file manually for synchronous access const content = fs.readFileSync(filename, 'utf-8'); const profileMatch = content.match(new RegExp(`\\[${profile}\\][\\s\\S]*?(?=\\n\\[|$)`)); - + if (!profileMatch) { throw new Error(`Profile "${profile}" not found in ${filename}`); } - + const accessKeyMatch = profileMatch[0].match(/aws_access_key_id\s*=\s*(.+)/); const secretKeyMatch = profileMatch[0].match(/aws_secret_access_key\s*=\s*(.+)/); - + if (!accessKeyMatch || !secretKeyMatch) { throw new Error(`Missing credentials in profile "${profile}"`); } @@ -36,25 +36,30 @@ function getAwsCredentials(profile, credFile = '/.aws/credentials') { } function getRealAwsConfig(location) { - const { awsEndpoint, gcpEndpoint, credentialsProfile, - credentials: locCredentials, bucketName, mpuBucketName, pathStyle } = - config.locationConstraints[location].details; + const { + awsEndpoint, + gcpEndpoint, + credentialsProfile, + credentials: locCredentials, + bucketName, + mpuBucketName, + pathStyle, + } = config.locationConstraints[location].details; const useHTTPS = config.locationConstraints[location].details.https; const proto = useHTTPS ? 'https' : 'http'; const isGcp = config.locationConstraints[location].type === 'gcp'; const params = { region: 'us-east-1', - endpoint: gcpEndpoint ? - `${proto}://${gcpEndpoint}` : `${proto}://${awsEndpoint}`, + endpoint: gcpEndpoint ? `${proto}://${gcpEndpoint}` : `${proto}://${awsEndpoint}`, }; - + if (isGcp) { params.disableS3ExpressSessionAuth = true; params.useGlobalEndpoint = false; params.s3DisableBodySigning = true; params.mainBucket = bucketName; params.mpuBucket = mpuBucketName; - } + } if (useHTTPS) { params.requestHandler = { httpsAgent: new https.Agent({ keepAlive: true }), @@ -64,25 +69,26 @@ function getRealAwsConfig(location) { httpAgent: new http.Agent({ keepAlive: true }), }; } - + if (pathStyle) { params.forcePathStyle = true; } - + if (!useHTTPS) { params.sslEnabled = false; } - + if (credentialsProfile) { const credentials = getAwsCredentials(credentialsProfile, '/.aws/credentials'); params.credentials = credentials; - + if (isGcp) { return { s3Params: params, bucketName, mpuBucket: mpuBucketName || bucketName, - credentials: { // For raw HTTP requests (GCP format) + credentials: { + // For raw HTTP requests (GCP format) accessKey: credentials.accessKeyId, secretKey: credentials.secretAccessKey, }, @@ -90,11 +96,11 @@ function getRealAwsConfig(location) { } return params; } - params.credentials = { + params.credentials = { accessKeyId: locCredentials.accessKey, secretAccessKey: locCredentials.secretKey, }; - + // For GCP with plain credentials, return nested structure if (isGcp) { return { @@ -113,7 +119,7 @@ function getRealAwsConfig(location) { }, }; } - + return params; } diff --git a/tests/functional/aws-node-sdk/test/support/objectConfigs.js b/tests/functional/aws-node-sdk/test/support/objectConfigs.js index ed2e9cb0bf..cf4324dbdf 100644 --- a/tests/functional/aws-node-sdk/test/support/objectConfigs.js +++ b/tests/functional/aws-node-sdk/test/support/objectConfigs.js @@ -10,9 +10,8 @@ const canonicalObjectConfig = { invalidPartNumbers: [-1, 0, maximumAllowedPartCount + 1], signature: 'for canonical object', meta: { - computeTotalSize: (partNumbers, bodySize) => partNumbers.reduce((total, current) => - total + bodySize + current + 1 - , 0), + computeTotalSize: (partNumbers, bodySize) => + partNumbers.reduce((total, current) => total + bodySize + current + 1, 0), objectIsEmpty: false, }, }; @@ -32,9 +31,6 @@ const emptyObjectConfig = { }, }; -const objectConfigs = [ - canonicalObjectConfig, - emptyObjectConfig, -]; +const objectConfigs = [canonicalObjectConfig, emptyObjectConfig]; module.exports = objectConfigs; diff --git a/tests/functional/aws-node-sdk/test/support/withV4.js b/tests/functional/aws-node-sdk/test/support/withV4.js index e875ed68f2..03dd53c3f0 100644 --- a/tests/functional/aws-node-sdk/test/support/withV4.js +++ b/tests/functional/aws-node-sdk/test/support/withV4.js @@ -12,11 +12,13 @@ function withV4(testFn) { config = {}; } - describe(`With ${version} signature`, (cfg => - function tcWrap() { - testFn.call(this, cfg); - } - )(config)); + describe( + `With ${version} signature`, + (cfg => + function tcWrap() { + testFn.call(this, cfg); + })(config), + ); }); } diff --git a/tests/functional/aws-node-sdk/test/utils/init.js b/tests/functional/aws-node-sdk/test/utils/init.js index 1cfd6a67a8..c39fe7b4c0 100644 --- a/tests/functional/aws-node-sdk/test/utils/init.js +++ b/tests/functional/aws-node-sdk/test/utils/init.js @@ -4,12 +4,9 @@ const metadata = require('../../../../../lib/metadata/wrapper'); const { config } = require('../../../../../lib/Config'); const { DummyRequestLogger } = require('../../../../unit/helpers'); const log = new DummyRequestLogger(); -const nonVersionedObjId = - versionIdUtils.getInfVid(config.replicationGroupId); +const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId); -const { - LOCATION_NAME_DMF, -} = require('../../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../../constants'); const isMetadataOrFile = ['file', 'scality'].includes(config.backends.metadata); /** @@ -57,13 +54,11 @@ function initMetadata(cb) { } function getMetadata(bucketName, objectName, versionId, cb) { - const promise = new Promise((resolve, reject) => metadata.getObjectMD( - bucketName, - objectName, - { versionId: decodeVersionId(versionId) }, - log, - (err, data) => (err ? reject(err) : resolve(data)), - )); + const promise = new Promise((resolve, reject) => + metadata.getObjectMD(bucketName, objectName, { versionId: decodeVersionId(versionId) }, log, (err, data) => + err ? reject(err) : resolve(data), + ), + ); return cb ? promise.then(res => cb(null, res), cb) : promise; } @@ -80,14 +75,11 @@ function fakeMetadataTransition(bucketName, objectName, versionId, cb) { const promise = (async () => { const objMD = await getMetadata(bucketName, objectName, versionId); objMD['x-amz-scal-transition-in-progress'] = true; - await new Promise((resolve, reject) => metadata.putObjectMD( - bucketName, - objectName, - objMD, - { versionId: decodeVersionId(versionId) }, - log, - err => (err ? reject(err) : resolve()), - )); + await new Promise((resolve, reject) => + metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, log, err => + err ? reject(err) : resolve(), + ), + ); })(); return cb ? promise.then(() => cb(), cb) : promise; } @@ -108,14 +100,11 @@ function fakeMetadataArchive(bucketName, objectName, versionId, archive, cb) { objMD['x-amz-storage-class'] = LOCATION_NAME_DMF; objMD.dataStoreName = LOCATION_NAME_DMF; objMD.archive = archive; - await new Promise((resolve, reject) => metadata.putObjectMD( - bucketName, - objectName, - objMD, - { versionId: decodeVersionId(versionId) }, - log, - err => (err ? reject(err) : resolve()), - )); + await new Promise((resolve, reject) => + metadata.putObjectMD(bucketName, objectName, objMD, { versionId: decodeVersionId(versionId) }, log, err => + err ? reject(err) : resolve(), + ), + ); })(); return cb ? promise.then(() => cb(), cb) : promise; } diff --git a/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js b/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js index 90dc8315d6..0dc39ac5d7 100644 --- a/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js +++ b/tests/functional/aws-node-sdk/test/versioning/bucketDelete.js @@ -15,7 +15,6 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util.js'); const bucketName = `versioning-bucket-${Date.now()}`; const key = 'anObject'; - function checkError(err, code) { assert.notEqual(err, null, 'Expected failure but got success'); assert.strictEqual(err.Code, code); @@ -29,12 +28,14 @@ describe('aws-node-sdk test delete bucket', () => { // setup test beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); }); // empty and delete bucket after testing if bucket exists @@ -43,48 +44,53 @@ describe('aws-node-sdk test delete bucket', () => { if (err?.name === 'NoSuchBucket') { return done(); } - return s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => done()).catch(err => { - if (err.name === 'NoSuchBucket') { - return done(); - } - return done(err); - }); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => done()) + .catch(err => { + if (err.name === 'NoSuchBucket') { + return done(); + } + return done(err); + }); }); }); - it('should be able to delete empty bucket with version enabled', - async () => { + it('should be able to delete empty bucket with version enabled', async () => { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should return error 409 BucketNotEmpty if trying to delete bucket' + - ' containing delete marker', async () => { - await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); - - try { - await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); - assert.fail('Expected BucketNotEmpty error but got success'); - } catch (err) { - checkError(err, 'BucketNotEmpty'); - } - }); + it( + 'should return error 409 BucketNotEmpty if trying to delete bucket' + ' containing delete marker', + async () => { + await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); - it('should return error 409 BucketNotEmpty if trying to delete bucket' + - ' containing version and delete marker', async () => { - await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); - await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); - - try { - await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); - assert.fail('Expected BucketNotEmpty error but got success'); - } catch (err) { - checkError(err, 'BucketNotEmpty'); - } - }); + try { + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); + assert.fail('Expected BucketNotEmpty error but got success'); + } catch (err) { + checkError(err, 'BucketNotEmpty'); + } + }, + ); + + it( + 'should return error 409 BucketNotEmpty if trying to delete bucket' + + ' containing version and delete marker', + async () => { + await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); + await s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: key })); + + try { + await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); + assert.fail('Expected BucketNotEmpty error but got success'); + } catch (err) { + checkError(err, 'BucketNotEmpty'); + } + }, + ); - it('should return error 404 NoSuchBucket if the bucket name is invalid', - async () => { + it('should return error 404 NoSuchBucket if the bucket name is invalid', async () => { try { await s3.send(new DeleteBucketCommand({ Bucket: 'bucketA' })); assert.fail('Expected NoSuchBucket error but got success'); diff --git a/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js b/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js index b773f77d77..d947e52de9 100644 --- a/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js +++ b/tests/functional/aws-node-sdk/test/versioning/legacyNullVersionCompat.js @@ -16,10 +16,7 @@ const { const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util.js'); // This series of tests can only be enabled on an environment that has // two Cloudserver instances, with one of them in null version @@ -29,8 +26,9 @@ const { // combination of Cloudserver requests to bucketd and the behavior of // bucketd based on those requests. -const describeSkipIfNotExplicitlyEnabled = - process.env.ENABLE_LEGACY_NULL_VERSION_COMPAT_TESTS ? describe : describe.skip; +const describeSkipIfNotExplicitlyEnabled = process.env.ENABLE_LEGACY_NULL_VERSION_COMPAT_TESTS + ? describe + : describe.skip; describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () => { const bucketUtilCompat = new BucketUtility('default', { @@ -46,25 +44,46 @@ describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () // master and no "isNull2" metadata attribute), by using the // Cloudserver endpoint that is configured with null version // compatibility mode enabled. - beforeEach(done => async.series([ - next => s3Compat.send(new CreateBucketCommand({ - Bucket: bucket, - }), next), - next => s3Compat.send(new PutObjectCommand({ - Bucket: bucket, - Key: 'obj', - Body: 'nullbody', - }), next), - next => s3Compat.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - }), next), - next => s3Compat.send(new PutObjectCommand({ - Bucket: bucket, - Key: 'obj', - Body: 'versionedbody', - }), next), - ], done)); + beforeEach(done => + async.series( + [ + next => + s3Compat.send( + new CreateBucketCommand({ + Bucket: bucket, + }), + next, + ), + next => + s3Compat.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'obj', + Body: 'nullbody', + }), + next, + ), + next => + s3Compat.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + next, + ), + next => + s3Compat.send( + new PutObjectCommand({ + Bucket: bucket, + Key: 'obj', + Body: 'versionedbody', + }), + next, + ), + ], + done, + ), + ); afterEach(done => { removeAllVersions({ Bucket: bucket }, err => { @@ -76,37 +95,56 @@ describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () }); it('updating ACL of legacy null version with non-compat cloudserver', done => { - async.series([ - next => s3.send(new PutObjectAclCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - ACL: 'public-read', - }), next), - next => s3.send(new GetObjectAclCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), (err, acl) => { - assert.ifError(err); - // check that we fetched the updated null version - assert.strictEqual(acl.Grants.length, 2); - next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), next), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - }), (err, listing) => { - assert.ifError(err); - // check that the null version has been correctly deleted - assert(listing.Versions.every(version => version.VersionId !== 'null')); - next(); - }), - ], done); + async.series( + [ + next => + s3.send( + new PutObjectAclCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + ACL: 'public-read', + }), + next, + ), + next => + s3.send( + new GetObjectAclCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + (err, acl) => { + assert.ifError(err); + // check that we fetched the updated null version + assert.strictEqual(acl.Grants.length, 2); + next(); + }, + ), + next => + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + next, + ), + next => + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + (err, listing) => { + assert.ifError(err); + // check that the null version has been correctly deleted + assert(listing.Versions.every(version => version.VersionId !== 'null')); + next(); + }, + ), + ], + done, + ); }); it('updating tags of legacy null version with non-compat cloudserver', done => { @@ -116,54 +154,81 @@ describeSkipIfNotExplicitlyEnabled('legacy null version compatibility tests', () Value: 'newtagvalue', }, ]; - async.series([ - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - Tagging: { - TagSet: tagSet, - }, - }), next), - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), (err, tagging) => { - assert.ifError(err); - assert.deepStrictEqual(tagging.TagSet, tagSet); - next(); - }), - next => s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), err => { - assert.ifError(err); - next(); - }), - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), (err, tagging) => { - assert.ifError(err); - assert.deepStrictEqual(tagging.TagSet, []); - next(); - }), - next => s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: 'obj', - VersionId: 'null', - }), next), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - }), (err, listing) => { - assert.ifError(err); - // check that the null version has been correctly deleted - assert(listing.Versions.every(version => version.VersionId !== 'null')); - next(); - }), - ], done); + async.series( + [ + next => + s3.send( + new PutObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + Tagging: { + TagSet: tagSet, + }, + }), + next, + ), + next => + s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + (err, tagging) => { + assert.ifError(err); + assert.deepStrictEqual(tagging.TagSet, tagSet); + next(); + }, + ), + next => + s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + err => { + assert.ifError(err); + next(); + }, + ), + next => + s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + (err, tagging) => { + assert.ifError(err); + assert.deepStrictEqual(tagging.TagSet, []); + next(); + }, + ), + next => + s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: 'obj', + VersionId: 'null', + }), + next, + ), + next => + s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + }), + (err, listing) => { + assert.ifError(err); + // check that the null version has been correctly deleted + assert(listing.Versions.every(version => version.VersionId !== 'null')); + next(); + }, + ), + ], + done, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js b/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js index d8a08ba99f..46fb7db8db 100644 --- a/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js +++ b/tests/functional/aws-node-sdk/test/versioning/listObjectMasterVersions.js @@ -16,22 +16,13 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util'); const bucket = `versioning-bucket-${Date.now()}`; const itSkipIfE2E = process.env.S3_END_TO_END ? it.skip : it; - function _assertResultElements(entry) { - const elements = [ - 'LastModified', - 'ETag', - 'Size', - 'Owner', - 'StorageClass', - ]; + const elements = ['LastModified', 'ETag', 'Size', 'Owner', 'StorageClass']; elements.forEach(elem => { - assert.notStrictEqual(entry[elem], undefined, - `Expected ${elem} in result but did not find it`); + assert.notStrictEqual(entry[elem], undefined, `Expected ${elem} in result but did not find it`); if (elem === 'Owner') { assert(entry.Owner.ID, 'Expected Owner ID but did not find it'); - assert(entry.Owner.DisplayName, - 'Expected Owner DisplayName but did not find it'); + assert(entry.Owner.DisplayName, 'Expected Owner DisplayName but did not find it'); } }); } @@ -53,8 +44,10 @@ describe('listObject - Delimiter master', function testSuite() { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); @@ -107,17 +100,21 @@ describe('listObject - Delimiter master', function testSuite() { } if (obj.value === null) { - const result = await s3.send(new DeleteObjectCommand({ + const result = await s3.send( + new DeleteObjectCommand({ Bucket: bucket, Key: obj.name, - })); - assert.strictEqual(result.DeleteMarker, true, 'Expected delete marker to be true'); + }), + ); + assert.strictEqual(result.DeleteMarker, true, 'Expected delete marker to be true'); } else { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: obj.name, - Body: obj.value, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: obj.name, + Body: obj.value, + }), + ); } } }); @@ -168,11 +165,7 @@ describe('listObject - Delimiter master', function testSuite() { { name: 'with maxKeys', params: { MaxKeys: 3 }, - expectedResult: [ - 'Pâtisserie=中文-español-English', - 'notes/spring/1.txt', - 'notes/spring/march/1.txt', - ], + expectedResult: ['Pâtisserie=中文-español-English', 'notes/spring/1.txt', 'notes/spring/march/1.txt'], commonPrefix: [], isTruncated: true, nextMarker: undefined, @@ -198,9 +191,7 @@ describe('listObject - Delimiter master', function testSuite() { { name: 'with delimiter', params: { Delimiter: '/' }, - expectedResult: [ - 'Pâtisserie=中文-español-English', - ], + expectedResult: ['Pâtisserie=中文-español-English'], commonPrefix: ['notes/'], isTruncated: false, nextMarker: undefined, @@ -235,15 +226,8 @@ describe('listObject - Delimiter master', function testSuite() { { name: 'delimiter and prefix (related to #147)', params: { Delimiter: '/', Prefix: 'notes/' }, - expectedResult: [ - 'notes/year.txt', - 'notes/yore.rs', - ], - commonPrefix: [ - 'notes/spring/', - 'notes/summer/', - 'notes/zaphod/', - ], + expectedResult: ['notes/year.txt', 'notes/yore.rs'], + commonPrefix: ['notes/spring/', 'notes/summer/', 'notes/zaphod/'], isTruncated: false, nextMarker: undefined, }, @@ -329,20 +313,16 @@ describe('listObject - Delimiter master', function testSuite() { runTest(test.name, async () => { const expectedResult = test.expectedResult; const res = await s3.send(new ListObjectsCommand(Object.assign({ Bucket: bucket }, test.params))); - + res.Contents?.forEach(result => { - if (!expectedResult - .find(key => key === result.Key)) { - throw new Error('listing fail, ' + - `unexpected key ${result.Key}`); + if (!expectedResult.find(key => key === result.Key)) { + throw new Error('listing fail, ' + `unexpected key ${result.Key}`); } _assertResultElements(result); }); res.CommonPrefixes?.forEach(cp => { - if (!test.commonPrefix - .find(item => item === cp.Prefix)) { - throw new Error('listing fail, ' + - `unexpected prefix ${cp.Prefix}`); + if (!test.commonPrefix.find(item => item === cp.Prefix)) { + throw new Error('listing fail, ' + `unexpected prefix ${cp.Prefix}`); } }); assert.strictEqual(res.IsTruncated, test.isTruncated); diff --git a/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js b/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js index 1685ec5413..102b220429 100644 --- a/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js +++ b/tests/functional/aws-node-sdk/test/versioning/listObjectVersions.js @@ -17,34 +17,21 @@ const { removeAllVersions } = require('../../lib/utility/versioning-util'); const bucket = `versioning-bucket-${Date.now()}`; const removeAllVersionsAsync = promisify(removeAllVersions); -const resultElements = [ - 'VersionId', - 'IsLatest', - 'LastModified', - 'Owner', -]; -const versionResultElements = [ - 'ETag', - 'Size', - 'StorageClass', -]; +const resultElements = ['VersionId', 'IsLatest', 'LastModified', 'Owner']; +const versionResultElements = ['ETag', 'Size', 'StorageClass']; function _assertResultElements(entry, type) { - const elements = type === 'DeleteMarker' ? resultElements : - resultElements.concat(versionResultElements); + const elements = type === 'DeleteMarker' ? resultElements : resultElements.concat(versionResultElements); elements.forEach(elem => { - assert.notStrictEqual(entry[elem], undefined, - `Expected ${elem} in result but did not find it`); + assert.notStrictEqual(entry[elem], undefined, `Expected ${elem} in result but did not find it`); if (elem === 'Owner') { assert(entry.Owner.ID, 'Expected Owner ID but did not find it'); - assert(entry.Owner.DisplayName, - 'Expected Owner DisplayName but did not find it'); + assert(entry.Owner.DisplayName, 'Expected Owner DisplayName but did not find it'); } }); } - describe('listObject - Delimiter version', function testSuite() { this.timeout(600000); @@ -91,33 +78,41 @@ describe('listObject - Delimiter version', function testSuite() { for (const obj of objects) { // Toggle bucket versioning state according to the original logic if (!versioning && obj.isNull !== true) { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Enabled' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); versioning = true; } else if (versioning && obj.isNull === true) { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: { Status: 'Suspended' }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); versioning = false; } if (obj.value === null) { // Create a delete marker, capture headers as in original test - const delRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: obj.name, - })); + const delRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: obj.name, + }), + ); assert.strictEqual(String(delRes.DeleteMarker), 'true'); obj.versionId = delRes.VersionId; } else { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: obj.name, - Body: obj.value, - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: obj.name, + Body: obj.value, + }), + ); obj.versionId = putRes.VersionId || 'null'; } } @@ -171,11 +166,7 @@ describe('listObject - Delimiter version', function testSuite() { { name: 'with maxKeys', params: { MaxKeys: 3 }, - expectedResult: [ - objects[4], - objects[5], - objects[8], - ], + expectedResult: [objects[4], objects[5], objects[8]], commonPrefix: [], isTruncated: true, nextKeyMarker: objects[8].name, @@ -202,8 +193,7 @@ describe('listObject - Delimiter version', function testSuite() { { name: 'with long delimiter', params: { Delimiter: 'notes/summer' }, - expectedResult: objects.filter(obj => - obj.name.indexOf('notes/summer') < 0), + expectedResult: objects.filter(obj => obj.name.indexOf('notes/summer') < 0), commonPrefix: ['notes/summer'], isTruncated: false, nextKeyMarker: undefined, @@ -225,15 +215,8 @@ describe('listObject - Delimiter version', function testSuite() { { name: 'delimiter and prefix (related to #147)', params: { Delimiter: '/', Prefix: 'notes/' }, - expectedResult: [ - objects[1], - objects[2], - ], - commonPrefix: [ - 'notes/spring/', - 'notes/summer/', - 'notes/zaphod/', - ], + expectedResult: [objects[1], objects[2]], + commonPrefix: ['notes/spring/', 'notes/summer/', 'notes/zaphod/'], isTruncated: false, nextKeyMarker: undefined, nextVersionIdMarker: undefined, @@ -324,59 +307,52 @@ describe('listObject - Delimiter version', function testSuite() { ].forEach(test => { it(test.name, async () => { const expectedResult = test.expectedResult; - const res = await s3.send(new ListObjectVersionsCommand({ - Bucket: bucket, - ...test.params, - })); + const res = await s3.send( + new ListObjectVersionsCommand({ + Bucket: bucket, + ...test.params, + }), + ); (res.Versions || []).forEach(result => { - const item = expectedResult.find(obj => ( - obj.name === result.Key - && obj.versionId === result.VersionId - && obj.value !== null - )); + const item = expectedResult.find( + obj => obj.name === result.Key && obj.versionId === result.VersionId && obj.value !== null, + ); if (!item) { - throw new Error('listing fail, ' - + `unexpected key ${result.Key} ` - + `with version ${result.VersionId}`); + throw new Error( + 'listing fail, ' + `unexpected key ${result.Key} ` + `with version ${result.VersionId}`, + ); } _assertResultElements(result, 'Version'); }); (res.DeleteMarkers || []).forEach(result => { - const item = expectedResult.find(obj => ( - obj.name === result.Key - && obj.versionId === result.VersionId - && obj.value === null - )); + const item = expectedResult.find( + obj => obj.name === result.Key && obj.versionId === result.VersionId && obj.value === null, + ); if (!item) { - throw new Error('listing fail, ' - + `unexpected key ${result.Key} ` - + `with version ${result.VersionId}`); + throw new Error( + 'listing fail, ' + `unexpected key ${result.Key} ` + `with version ${result.VersionId}`, + ); } _assertResultElements(result, 'DeleteMarker'); }); (res.CommonPrefixes || []).forEach(cp => { - if (!test.commonPrefix.find( - item => item === cp.Prefix, - )) { - throw new Error('listing fail, ' - + `unexpected prefix ${cp.Prefix}`); + if (!test.commonPrefix.find(item => item === cp.Prefix)) { + throw new Error('listing fail, ' + `unexpected prefix ${cp.Prefix}`); } }); assert.strictEqual(res.IsTruncated, test.isTruncated); - assert.strictEqual(res.NextKeyMarker, - test.nextKeyMarker); + assert.strictEqual(res.NextKeyMarker, test.nextKeyMarker); if (!test.nextVersionIdMarker) { // eslint-disable-next-line no-param-reassign test.nextVersionIdMarker = {}; } - assert.strictEqual(res.NextVersionIdMarker, - test.nextVersionIdMarker.versionId); + assert.strictEqual(res.NextVersionIdMarker, test.nextVersionIdMarker.versionId); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js b/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js index bf5d43d09e..8c50438e83 100644 --- a/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js +++ b/tests/functional/aws-node-sdk/test/versioning/multiObjectDelete.js @@ -17,9 +17,9 @@ const bucketName = `multi-object-delete-${Date.now()}`; const key = 'key'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; function sortList(list) { return list.sort((a, b) => { @@ -33,7 +33,6 @@ function sortList(list) { }); } - describe('Multi-Object Versioning Delete Success', function success() { this.timeout(360000); @@ -44,12 +43,14 @@ describe('Multi-Object Versioning Delete Success', function success() { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); const objects = []; for (let i = 1; i < 1001; i++) { @@ -65,8 +66,9 @@ describe('Multi-Object Versioning Delete Success', function success() { return await s3.send(new PutObjectCommand(params)); } catch (err) { if (attempt < 3) { - process.stdout.write(`Retrying PutObject ${params.Key} ` - + `(attempt ${attempt + 1}/3): ${err}\n`); + process.stdout.write( + `Retrying PutObject ${params.Key} ` + `(attempt ${attempt + 1}/3): ${err}\n`, + ); return putWithRetry(params, attempt + 1); } throw err; @@ -90,192 +92,215 @@ describe('Multi-Object Versioning Delete Success', function success() { objectsRes = results; }); - afterEach(done => { removeAllVersions({ Bucket: bucketName }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => done()) + .catch(done); }); }); it('should batch delete 1000 objects quietly', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: true, - }, - })); + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: true, + }, + }), + ); assert.strictEqual(res.Deleted, undefined); assert.strictEqual(res.Errors, undefined); }); it('should batch delete 1000 objects', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - Quiet: false, - }, - })); + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + Quiet: false, + }, + }), + ); assert.strictEqual(res.Deleted.length, 1000); // order of returned objects not sorted - assert.deepStrictEqual(sortList(res.Deleted), - sortList(objects)); + assert.deepStrictEqual(sortList(res.Deleted), sortList(objects)); assert.strictEqual(res.Errors, undefined); }); - it('should return NoSuchVersion in errors if one versionId is ' + - 'invalid', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); + it('should return NoSuchVersion in errors if one versionId is ' + 'invalid', async () => { + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); objects[0].VersionId = 'invalid-version-id'; - - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })); + + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ); assert.strictEqual(res.Deleted.length, 999); assert.strictEqual(res.Errors.length, 1); assert.strictEqual(res.Errors[0].Code, 'NoSuchVersion'); }); - it('should not send back any error if a versionId does not exist ' + - 'and should not create a new delete marker', async () => { - const objects = objectsRes.slice(0, 1000).map(obj => - ({ Key: obj.Key, VersionId: obj.VersionId })); - objects[0].VersionId = nonExistingId; - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })); - assert.strictEqual(res.Deleted.length, 1000); - assert.strictEqual(res.Errors, undefined); - const foundVersionId = res.Deleted.find(entry => - entry.VersionId === nonExistingId); - assert(foundVersionId); - assert.strictEqual(foundVersionId.DeleteMarker, undefined); - }); + it( + 'should not send back any error if a versionId does not exist ' + + 'and should not create a new delete marker', + async () => { + const objects = objectsRes.slice(0, 1000).map(obj => ({ Key: obj.Key, VersionId: obj.VersionId })); + objects[0].VersionId = nonExistingId; + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ); + assert.strictEqual(res.Deleted.length, 1000); + assert.strictEqual(res.Errors, undefined); + const foundVersionId = res.Deleted.find(entry => entry.VersionId === nonExistingId); + assert(foundVersionId); + assert.strictEqual(foundVersionId.DeleteMarker, undefined); + }, + ); it('should not crash when deleting a null versionId that does not exist', async () => { const objects = [{ Key: objectsRes[0].Key, VersionId: 'null' }]; - const res = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: objects, - }, - })); + const res = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: objects, + }, + }), + ); assert.deepStrictEqual(res.Deleted, [{ Key: objectsRes[0].Key, VersionId: 'null' }]); assert.strictEqual(res.Errors, undefined); }); }); }); -describe('Multi-Object Versioning Delete - deleting delete marker', -() => { +describe('Multi-Object Versioning Delete - deleting delete marker', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ); }); - afterEach(done => { + afterEach(done => { removeAllVersions({ Bucket: bucketName }, err => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucketName })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucketName })) + .then(() => done()) + .catch(done); }); }); - it('should send back VersionId and DeleteMarkerVersionId both equal ' + - 'to deleteVersionId', async () => { + it('should send back VersionId and DeleteMarkerVersionId both equal ' + 'to deleteVersionId', async () => { await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); - - const deleteRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: key - })); + + const deleteRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: key, + }), + ); const deleteVersionId = deleteRes.VersionId; - const deleteObjectsRes = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { - Key: key, - VersionId: deleteVersionId, - }, - ], - } - })); + const deleteObjectsRes = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: key, + VersionId: deleteVersionId, + }, + ], + }, + }), + ); assert.strictEqual(deleteObjectsRes.Deleted[0].DeleteMarker, true); assert.strictEqual(deleteObjectsRes.Deleted[0].VersionId, deleteVersionId); assert.strictEqual(deleteObjectsRes.Deleted[0].DeleteMarkerVersionId, deleteVersionId); }); - it('should send back a DeleteMarkerVersionId matching the versionId ' + - 'stored for the object if trying to delete an object that does not exist', async () => { - const deleteRes = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { - Key: key, + it( + 'should send back a DeleteMarkerVersionId matching the versionId ' + + 'stored for the object if trying to delete an object that does not exist', + async () => { + const deleteRes = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: key, + }, + ], }, - ], - } - })); + }), + ); - const versionIdFromDeleteObjects = deleteRes.Deleted[0].DeleteMarkerVersionId; - assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); + const versionIdFromDeleteObjects = deleteRes.Deleted[0].DeleteMarkerVersionId; + assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); - const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); - const versionIdFromListObjectVersions = listRes.DeleteMarkers[0].VersionId; - assert.strictEqual(versionIdFromDeleteObjects, versionIdFromListObjectVersions); - }); + const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); + const versionIdFromListObjectVersions = listRes.DeleteMarkers[0].VersionId; + assert.strictEqual(versionIdFromDeleteObjects, versionIdFromListObjectVersions); + }, + ); - it('should send back a DeleteMarkerVersionId matching the versionId ' + - 'stored for the object if object exists but no version was specified', async () => { - const putRes = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); - const versionId = putRes.VersionId; - const deleteRes = await s3.send(new DeleteObjectsCommand({ - Bucket: bucketName, - Delete: { - Objects: [ - { - Key: key, + it( + 'should send back a DeleteMarkerVersionId matching the versionId ' + + 'stored for the object if object exists but no version was specified', + async () => { + const putRes = await s3.send(new PutObjectCommand({ Bucket: bucketName, Key: key })); + const versionId = putRes.VersionId; + const deleteRes = await s3.send( + new DeleteObjectsCommand({ + Bucket: bucketName, + Delete: { + Objects: [ + { + Key: key, + }, + ], }, - ], - } - })); + }), + ); - assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); - const deleteVersionId = deleteRes.Deleted[0].DeleteMarkerVersionId; - assert.notEqual(deleteVersionId, versionId); + assert.strictEqual(deleteRes.Deleted[0].DeleteMarker, true); + const deleteVersionId = deleteRes.Deleted[0].DeleteMarkerVersionId; + assert.notEqual(deleteVersionId, versionId); - const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); - assert.strictEqual(deleteVersionId, listRes.DeleteMarkers[0].VersionId); - assert.strictEqual(versionId, listRes.Versions[0].VersionId); - }); + const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucketName })); + assert.strictEqual(deleteVersionId, listRes.DeleteMarkers[0].VersionId); + assert.strictEqual(versionId, listRes.Versions[0].VersionId); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectACL.js b/tests/functional/aws-node-sdk/test/versioning/objectACL.js index b03e5a3ccf..2f65a19cb1 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectACL.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectACL.js @@ -26,9 +26,9 @@ const key = '/'; const invalidId = 'invalidIdWithMoreThan40BytesAndThatIsNotLongEnoughYet'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; class _Utils { constructor(s3) { @@ -44,25 +44,30 @@ class _Utils { async _wrapDataObject(method, params) { const Command = method === 'getObjectAcl' ? GetObjectAclCommand : PutObjectAclCommand; const data = await this.s3.send(new Command(params)); - + let versionId = params.VersionId; - + if (!versionId) { // For non-version-specific ACL operations, we need to determine the latest version try { - const headResult = await this.s3.send(new HeadObjectCommand({ - Bucket: params.Bucket, - Key: params.Key - })); + const headResult = await this.s3.send( + new HeadObjectCommand({ + Bucket: params.Bucket, + Key: params.Key, + }), + ); versionId = headResult.VersionId; } catch { versionId = undefined; // Fallback } } - - const dataObj = Object.assign({ - VersionId: versionId, - }, data); + + const dataObj = Object.assign( + { + VersionId: versionId, + }, + data, + ); return dataObj; } @@ -83,18 +88,20 @@ class _Utils { if (versionId) { params.VersionId = versionId; } - + try { const data = await this.putObjectAcl(params); if (expected.error) { // Should not reach here if error was expected assert.fail('Expected error but operation succeeded'); } - _Utils.assertNoError(null, - `putting object acl with version id: ${versionId}`); - assert.strictEqual(data.VersionId, expected.versionId, + _Utils.assertNoError(null, `putting object acl with version id: ${versionId}`); + assert.strictEqual( + data.VersionId, + expected.versionId, `expected version id '${expected.versionId}' in ` + - `putacl res headers, got '${data.VersionId}' instead`); + `putacl res headers, got '${data.VersionId}' instead`, + ); } catch (err) { if (expected.error) { assert.strictEqual(expected.error.code, err.Code); @@ -103,19 +110,20 @@ class _Utils { throw err; } } - + delete params.ACL; - + try { const data = await this.getObjectAcl(params); if (expected.error) { assert.fail('Expected error but operation succeeded'); } - _Utils.assertNoError(null, - `getting object acl with version id: ${versionId}`); - assert.strictEqual(data.VersionId, expected.versionId, - `expected version id '${expected.versionId}' in ` + - `getacl res headers, got '${data.VersionId}'`); + _Utils.assertNoError(null, `getting object acl with version id: ${versionId}`); + assert.strictEqual( + data.VersionId, + expected.versionId, + `expected version id '${expected.versionId}' in ` + `getacl res headers, got '${data.VersionId}'`, + ); assert.strictEqual(data.Grants.length, 2); } catch (err) { if (expected.error) { @@ -131,111 +139,124 @@ class _Utils { function _testBehaviorVersioningEnabledOrSuspended(utils, versionIds) { const s3 = utils.s3; - it('should return 405 MethodNotAllowed putting acl without ' + - 'version id if latest version is a delete marker', async () => { - const aclParams = { - Bucket: bucket, - Key: key, - ACL: 'public-read-write', - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - - try { - await utils.putObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'MethodNotAllowed'); - assert.strictEqual(err.$metadata.httpStatusCode, 405); - } - }); - - it('should return 405 MethodNotAllowed putting acl with ' + - 'version id if version specified is a delete marker', async () => { - const aclParams = { - Bucket: bucket, - Key: key, - ACL: 'public-read-write', - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - aclParams.VersionId = data.VersionId; - - try { - await utils.putObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'MethodNotAllowed'); - assert.strictEqual(err.$metadata.httpStatusCode, 405); - } - }); + it( + 'should return 405 MethodNotAllowed putting acl without ' + 'version id if latest version is a delete marker', + async () => { + const aclParams = { + Bucket: bucket, + Key: key, + ACL: 'public-read-write', + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); - it('should return 404 NoSuchKey getting acl without ' + - 'version id if latest version is a delete marker', async () => { - const aclParams = { - Bucket: bucket, - Key: key, - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - - try { - await utils.getObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'NoSuchKey'); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - } - }); + try { + await utils.putObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.strictEqual(err.$metadata.httpStatusCode, 405); + } + }, + ); + + it( + 'should return 405 MethodNotAllowed putting acl with ' + 'version id if version specified is a delete marker', + async () => { + const aclParams = { + Bucket: bucket, + Key: key, + ACL: 'public-read-write', + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); + aclParams.VersionId = data.VersionId; - it('should return 405 MethodNotAllowed getting acl with ' + - 'version id if version specified is a delete marker', async () => { - const latestVersion = versionIds[versionIds.length - 1]; - const aclParams = { - Bucket: bucket, - Key: key, - VersionId: latestVersion, - }; - const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); - assert.strictEqual(data.DeleteMarker, true); - assert(data.VersionId); - aclParams.VersionId = data.VersionId; - - try { - await utils.getObjectAcl(aclParams); - assert.fail('Expected error but operation succeeded'); - } catch (err) { - assert(err); - assert.strictEqual(err.Code, 'MethodNotAllowed'); - assert.strictEqual(err.$metadata.httpStatusCode, 405); - } - }); + try { + await utils.putObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.strictEqual(err.$metadata.httpStatusCode, 405); + } + }, + ); + + it( + 'should return 404 NoSuchKey getting acl without ' + 'version id if latest version is a delete marker', + async () => { + const aclParams = { + Bucket: bucket, + Key: key, + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); - it('non-version specific put and get ACL should target latest ' + - 'version AND return version ID in response headers', async () => { - const latestVersion = versionIds[versionIds.length - 1]; - const expectedRes = { versionId: latestVersion }; - await utils.putAndGetAcl('public-read', undefined, expectedRes); - }); + try { + await utils.getObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'NoSuchKey'); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + } + }, + ); + + it( + 'should return 405 MethodNotAllowed getting acl with ' + 'version id if version specified is a delete marker', + async () => { + const latestVersion = versionIds[versionIds.length - 1]; + const aclParams = { + Bucket: bucket, + Key: key, + VersionId: latestVersion, + }; + const data = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); + assert.strictEqual(data.DeleteMarker, true); + assert(data.VersionId); + aclParams.VersionId = data.VersionId; - it('version specific put and get ACL should return version ID ' + - 'in response headers', async () => { + try { + await utils.getObjectAcl(aclParams); + assert.fail('Expected error but operation succeeded'); + } catch (err) { + assert(err); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.strictEqual(err.$metadata.httpStatusCode, 405); + } + }, + ); + + it( + 'non-version specific put and get ACL should target latest ' + + 'version AND return version ID in response headers', + async () => { + const latestVersion = versionIds[versionIds.length - 1]; + const expectedRes = { versionId: latestVersion }; + await utils.putAndGetAcl('public-read', undefined, expectedRes); + }, + ); + + it('version specific put and get ACL should return version ID ' + 'in response headers', async () => { const firstVersion = versionIds[0]; const expectedRes = { versionId: firstVersion }; await utils.putAndGetAcl('public-read', firstVersion, expectedRes); }); - it('version specific put and get ACL (version id = "null") ' + - 'should return version ID ("null") in response headers', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', 'null', expectedRes); - }); + it( + 'version specific put and get ACL (version id = "null") ' + + 'should return version ID ("null") in response headers', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', 'null', expectedRes); + }, + ); } describe('versioned put and get object acl ::', () => { @@ -254,8 +275,10 @@ describe('versioned put and get object acl ::', () => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) - .then(() => done()).catch(done); + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) + .then(() => done()) + .catch(done); }); }); @@ -264,42 +287,45 @@ describe('versioned put and get object acl ::', () => { await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); }); - it('should not return version id for non-version specific ' + - 'put and get ACL', async () => { + it('should not return version id for non-version specific ' + 'put and get ACL', async () => { const expectedRes = { versionId: undefined }; await utils.putAndGetAcl('public-read', undefined, expectedRes); }); - it('should not return version id for version specific ' + - 'put and get ACL (version id = "null")', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', 'null', expectedRes); - }); - - it('should return NoSuchVersion if attempting to put or get acl ' + - 'for non-existing version', async () => { - const error = { code: 'NoSuchVersion', statusCode: 404 }; - await utils.putAndGetAcl('private', nonExistingId, { error }); - }); - - it('should return InvalidArgument if attempting to put/get acl ' + - 'for invalid hex string', async () => { + it( + 'should not return version id for version specific ' + 'put and get ACL (version id = "null")', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', 'null', expectedRes); + }, + ); + + it( + 'should return NoSuchVersion if attempting to put or get acl ' + 'for non-existing version', + async () => { + const error = { code: 'NoSuchVersion', statusCode: 404 }; + await utils.putAndGetAcl('private', nonExistingId, { error }); + }, + ); + + it('should return InvalidArgument if attempting to put/get acl ' + 'for invalid hex string', async () => { const error = { code: 'InvalidArgument', statusCode: 400 }; await utils.putAndGetAcl('private', invalidId, { error }); }); }); - describe('on a version-enabled bucket with non-versioned object :: ', - () => { + describe('on a version-enabled bucket with non-versioned object :: ', () => { const versionIds = []; beforeEach(async () => { const params = { Bucket: bucket, Key: key }; await s3.send(new PutObjectCommand(params)); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(() => { @@ -308,11 +334,14 @@ describe('versioned put and get object acl ::', () => { }); describe('before putting new versions :: ', () => { - it('non-version specific put and get ACL should now ' + - 'return version ID ("null") in response headers', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', undefined, expectedRes); - }); + it( + 'non-version specific put and get ACL should now ' + + 'return version ID ("null") in response headers', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', undefined, expectedRes); + }, + ); }); describe('after putting new versions :: ', () => { @@ -329,39 +358,43 @@ describe('versioned put and get object acl ::', () => { }); }); - describe('on a version-enabled bucket - version non-specified :: ', - () => { + describe('on a version-enabled bucket - version non-specified :: ', () => { let versionId; beforeEach(async () => { const params = { Bucket: bucket, Key: key }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); const data = await s3.send(new PutObjectCommand(params)); versionId = data.VersionId; }); - it('should not create version putting ACL on a' + - 'version-enabled bucket where no version id is specified', - async () => { - const params = { Bucket: bucket, Key: key, ACL: 'public-read' }; - await utils.putObjectAcl(params); - await checkOneVersion(s3, bucket, versionId); - }); + it( + 'should not create version putting ACL on a' + + 'version-enabled bucket where no version id is specified', + async () => { + const params = { Bucket: bucket, Key: key, ACL: 'public-read' }; + await utils.putObjectAcl(params); + await checkOneVersion(s3, bucket, versionId); + }, + ); }); - describe('on version-suspended bucket with non-versioned object :: ', - () => { + describe('on version-suspended bucket with non-versioned object :: ', () => { const versionIds = []; beforeEach(async () => { const params = { Bucket: bucket, Key: key }; await s3.send(new PutObjectCommand(params)); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); afterEach(() => { @@ -370,31 +403,38 @@ describe('versioned put and get object acl ::', () => { }); describe('before putting new versions :: ', () => { - it('non-version specific put and get ACL should still ' + - 'return version ID ("null") in response headers', async () => { - const expectedRes = { versionId: 'null' }; - await utils.putAndGetAcl('public-read', undefined, expectedRes); - }); + it( + 'non-version specific put and get ACL should still ' + + 'return version ID ("null") in response headers', + async () => { + const expectedRes = { versionId: 'null' }; + await utils.putAndGetAcl('public-read', undefined, expectedRes); + }, + ); }); describe('after putting new versions :: ', () => { beforeEach(async () => { const params = { Bucket: bucket, Key: key }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); - + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); + for (let i = 0; i < counter; i++) { const data = await s3.send(new PutObjectCommand(params)); _Utils.assertNoError(null, `putting version #${i}`); versionIds.push(data.VersionId); } - - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); _testBehaviorVersioningEnabledOrSuspended(utils, versionIds); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js index 7158aa0a42..e1b8aedf57 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectCopy.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectCopy.js @@ -22,7 +22,6 @@ const customS3Request = require('../../lib/utility/customS3Request'); const removeAllVersionsPromise = promisify(removeAllVersions); - const { taggingTests } = require('../../lib/utility/tagging'); const constants = require('../../../../../constants'); @@ -64,8 +63,7 @@ const otherAccountBucketUtility = new BucketUtility('lisa', {}); const otherAccountS3 = otherAccountBucketUtility.s3; function checkNoError(err) { - assert.equal(err, null, - `Expected success, got error ${JSON.stringify(err)}`); + assert.equal(err, null, `Expected success, got error ${JSON.stringify(err)}`); } function checkError(err, code) { @@ -102,52 +100,59 @@ describe('Object Version Copy', () => { beforeEach(async () => { await bucketUtil.createOne(sourceBucketName); await bucketUtil.createOne(destBucketName); - await s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - const putRes = await s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: content, - Metadata: originalMetadata, - CacheControl: originalCacheControl, - ContentDisposition: originalContentDisposition, - ContentEncoding: originalContentEncoding, - Expires: originalExpires, - Tagging: originalTagging, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: content, + Metadata: originalMetadata, + CacheControl: originalCacheControl, + ContentDisposition: originalContentDisposition, + ContentEncoding: originalContentEncoding, + Expires: originalExpires, + Tagging: originalTagging, + }), + ); etag = putRes.ETag; versionId = putRes.VersionId; copySource = `${sourceBucketName}/${sourceObjName}?versionId=${versionId}`; etagTrim = etag.substring(1, etag.length - 1); copySourceVersionId = putRes.VersionId; - const headRes = await s3.send(new HeadObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - })); + const headRes = await s3.send( + new HeadObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + }), + ); lastModified = headRes.LastModified; - await s3.send(new PutObjectCommand({ - Bucket: sourceBucketName, - Key: sourceObjName, - Body: secondContent, - })); + await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: secondContent, + }), + ); }); afterEach(async () => { - await Promise.all([ - emptyAndDeleteBucket(sourceBucketName), - emptyAndDeleteBucket(destBucketName), - ]); + await Promise.all([emptyAndDeleteBucket(sourceBucketName), emptyAndDeleteBucket(destBucketName)]); }); async function requestCopy(fields) { - return s3.send(new CopyObjectCommand({ - Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - ...fields, - })); + return s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + ...fields, + }), + ); } async function successCopyCheck(error, response, copyVersionMetadata, destBucketName, destObjName) { @@ -157,8 +162,7 @@ describe('Object Version Copy', () => { const destinationVersionId = response.VersionId; assert.strictEqual(response.CopyObjectResult.ETag, etag); const copyLastModified = new Date(response.CopyObjectResult.LastModified).toGMTString(); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); assert.strictEqual(res.VersionId, destinationVersionId); const responseBody = await res.Body.transformToString(); assert.strictEqual(responseBody, content); @@ -172,30 +176,51 @@ describe('Object Version Copy', () => { assert.strictEqual(data.TagSet[0].Value, value); } - it('should copy an object from a source bucket to a different '+ - 'destination bucket and copy the tag set if no tagging directive '+ - 'header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource })); - await checkSuccessTagging(originalTagKey, originalTagValue); - }); - - it('should copy an object from a source bucket to a different ' + - 'destination bucket and copy the tag set if COPY tagging ' + - 'directive header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - TaggingDirective: 'COPY' })); - await checkSuccessTagging(originalTagKey, originalTagValue); - }); - - it('should copy an object from a source to the same destination '+ - 'updating tag if REPLACE tagging directive header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - TaggingDirective: 'REPLACE', Tagging: newTagging })); - await checkSuccessTagging(newTagKey, newTagValue); - }); + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the tag set if no tagging directive ' + + 'header provided', + async () => { + await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + await checkSuccessTagging(originalTagKey, originalTagValue); + }, + ); + + it( + 'should copy an object from a source bucket to a different ' + + 'destination bucket and copy the tag set if COPY tagging ' + + 'directive header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + TaggingDirective: 'COPY', + }), + ); + await checkSuccessTagging(originalTagKey, originalTagValue); + }, + ); + + it( + 'should copy an object from a source to the same destination ' + + 'updating tag if REPLACE tagging directive header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + TaggingDirective: 'REPLACE', + Tagging: newTagging, + }), + ); + await checkSuccessTagging(newTagKey, newTagValue); + }, + ); describe('Copy object with versioning updating tag set', () => { taggingTests.forEach(taggingTest => { @@ -203,9 +228,13 @@ describe('Object Version Copy', () => { const key = encodeURIComponent(taggingTest.tag.key); const value = encodeURIComponent(taggingTest.tag.value); const tagging = `${key}=${value}`; - const params = { Bucket: destBucketName, Key: destObjName, CopySource: copySource, + const params = { + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, TaggingDirective: 'REPLACE', - Tagging: tagging }; + Tagging: tagging, + }; try { await s3.send(new CopyObjectCommand(params)); await checkSuccessTagging(taggingTest.tag.key, taggingTest.tag.value); @@ -232,8 +261,7 @@ describe('Object Version Copy', () => { } }); - it('should return InvalidArgument for a request with empty string '+ - 'versionId query', async () => { + it('should return InvalidArgument for a request with empty string ' + 'versionId query', async () => { const params = { Bucket: destBucketName, Key: destObjName, CopySource: copySource }; const query = { versionId: '' }; try { @@ -245,137 +273,193 @@ describe('Object Version Copy', () => { } }); - it('should copy a version from a source bucket to a different' + - 'destination bucket and copy the metadata if no metadata directive' + - 'header provided', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource })); - await successCopyCheck(null, res, originalMetadata, destBucketName, destObjName); - }); - - it('should also copy additional headers (CacheControl, ' + - 'ContentDisposition, ContentEncoding, Expires) when copying an ' + - 'object from a source bucket to a different destination bucket', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.CacheControl, originalCacheControl); - assert.strictEqual(res.ContentDisposition, originalContentDisposition); - assert.strictEqual(res.ContentEncoding, 'base64,'); - assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); - }); - - it('should copy an object from a source bucket to a different '+ - 'key in the same bucket', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, - Key: destObjName, - CopySource: copySource })); - await successCopyCheck(null, res, originalMetadata, - sourceBucketName, destObjName); - }); - - it('should copy an object from a source to the same destination ' + - '(update metadata)', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); + it( + 'should copy a version from a source bucket to a different' + + 'destination bucket and copy the metadata if no metadata directive' + + 'header provided', + async () => { + const res = await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + await successCopyCheck(null, res, originalMetadata, destBucketName, destObjName); + }, + ); + + it( + 'should also copy additional headers (CacheControl, ' + + 'ContentDisposition, ContentEncoding, Expires) when copying an ' + + 'object from a source bucket to a different destination bucket', + async () => { + await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.CacheControl, originalCacheControl); + assert.strictEqual(res.ContentDisposition, originalContentDisposition); + assert.strictEqual(res.ContentEncoding, 'base64,'); + assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); + }, + ); + + it('should copy an object from a source bucket to a different ' + 'key in the same bucket', async () => { + const res = await s3.send( + new CopyObjectCommand({ Bucket: sourceBucketName, Key: destObjName, CopySource: copySource }), + ); + await successCopyCheck(null, res, originalMetadata, sourceBucketName, destObjName); + }); + + it('should copy an object from a source to the same destination ' + '(update metadata)', async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); await successCopyCheck(null, res, newMetadata, sourceBucketName, sourceObjName); }); - it('should copy an object and replace the metadata if replace ' + - 'included as metadata directive header', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); - await successCopyCheck(null, res, newMetadata, destBucketName, destObjName); - }); - - it('should copy an object and replace ContentType if replace ' + - 'included as a metadata directive header, and new ContentType is ' + - 'provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - ContentType: 'image' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.ContentType, 'image'); - }); - - it('should copy an object and keep ContentType if replace ' + - 'included as a metadata directive header, but no new ContentType ' + - 'is provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.ContentType, 'application/octet-stream'); - }); - - it('should also replace additional headers if replace ' + - 'included as metadata directive header and new headers are ' + - 'specified', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - CacheControl: newCacheControl, - ContentDisposition: newContentDisposition, - ContentEncoding: newContentEncoding, - Expires: newExpires })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.CacheControl, newCacheControl); - assert.strictEqual(res.ContentDisposition, newContentDisposition); - assert.strictEqual(res.ContentEncoding, 'gzip,'); - assert.strictEqual(res.Expires.toGMTString(), newExpires.toGMTString()); - }); - - it('should copy an object and the metadata if copy ' + - 'included as metadata directive header (and ignore any new ' + - 'metadata sent with copy request)', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'COPY', - Metadata: newMetadata })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.deepStrictEqual(res.Metadata, originalMetadata); - }); - - it('should copy an object and its additional headers if copy ' + - 'included as metadata directive header (and ignore any new ' + - 'headers sent with copy request)', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'COPY', - Metadata: newMetadata, - CacheControl: newCacheControl, - ContentDisposition: newContentDisposition, - ContentEncoding: newContentEncoding, - Expires: newExpires })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.CacheControl, originalCacheControl); - assert.strictEqual(res.ContentDisposition, originalContentDisposition); - assert.strictEqual(res.ContentEncoding, 'base64,'); - assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); - }); + it( + 'should copy an object and replace the metadata if replace ' + 'included as metadata directive header', + async () => { + const res = await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); + await successCopyCheck(null, res, newMetadata, destBucketName, destObjName); + }, + ); + + it( + 'should copy an object and replace ContentType if replace ' + + 'included as a metadata directive header, and new ContentType is ' + + 'provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + ContentType: 'image', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ContentType, 'image'); + }, + ); + + it( + 'should copy an object and keep ContentType if replace ' + + 'included as a metadata directive header, but no new ContentType ' + + 'is provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ContentType, 'application/octet-stream'); + }, + ); + + it( + 'should also replace additional headers if replace ' + + 'included as metadata directive header and new headers are ' + + 'specified', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + CacheControl: newCacheControl, + ContentDisposition: newContentDisposition, + ContentEncoding: newContentEncoding, + Expires: newExpires, + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.CacheControl, newCacheControl); + assert.strictEqual(res.ContentDisposition, newContentDisposition); + assert.strictEqual(res.ContentEncoding, 'gzip,'); + assert.strictEqual(res.Expires.toGMTString(), newExpires.toGMTString()); + }, + ); + + it( + 'should copy an object and the metadata if copy ' + + 'included as metadata directive header (and ignore any new ' + + 'metadata sent with copy request)', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'COPY', + Metadata: newMetadata, + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.deepStrictEqual(res.Metadata, originalMetadata); + }, + ); + + it( + 'should copy an object and its additional headers if copy ' + + 'included as metadata directive header (and ignore any new ' + + 'headers sent with copy request)', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'COPY', + Metadata: newMetadata, + CacheControl: newCacheControl, + ContentDisposition: newContentDisposition, + ContentEncoding: newContentEncoding, + Expires: newExpires, + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.CacheControl, originalCacheControl); + assert.strictEqual(res.ContentDisposition, originalContentDisposition); + assert.strictEqual(res.ContentEncoding, 'base64,'); + assert.strictEqual(res.Expires.toGMTString(), originalExpires.toGMTString()); + }, + ); it('should copy a 0 byte object to different destination', async () => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - const putRes = await s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - Body: '', - Metadata: originalMetadata })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + Body: '', + Metadata: originalMetadata, + }), + ); copySource = `${sourceBucketName}/${sourceObjName}?versionId=${putRes.VersionId}`; - const copyRes = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource })); + const copyRes = await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); assert.strictEqual(copyRes.CopyObjectResult.ETag, emptyFileETag); - const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); + const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); assert.deepStrictEqual(getRes.Metadata, originalMetadata); assert.strictEqual(getRes.ETag, emptyFileETag); }); @@ -384,119 +468,181 @@ describe('Object Version Copy', () => { if (constants.validStorageClasses.includes('REDUCED_REDUNDANCY')) { it('should copy a 0 byte object to same destination', async () => { const emptyFileETag = '"d41d8cd98f00b204e9800998ecf8427e"'; - const putRes = await s3.send(new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - Body: '' })); + const putRes = await s3.send( + new PutObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, Body: '' }), + ); copySource = `${sourceBucketName}/${sourceObjName}?versionId=${putRes.VersionId}`; - const copyRes = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource, - StorageClass: 'REDUCED_REDUNDANCY' })); + const copyRes = await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: copySource, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ); assert.notEqual(copyRes.VersionId, putRes.VersionId); assert.strictEqual(copyRes.ETag, emptyFileETag); - const getRes = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })); + const getRes = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); assert.deepStrictEqual(getRes.Metadata, {}); - assert.strictEqual(getRes.StorageClass, - 'REDUCED_REDUNDANCY'); + assert.strictEqual(getRes.StorageClass, 'REDUCED_REDUNDANCY'); assert.strictEqual(getRes.ETag, emptyFileETag); }); - it('should copy an object to a different destination and change ' + - 'the storage class if storage class header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - StorageClass: 'REDUCED_REDUNDANCY' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); - }); - - it('should copy an object to the same destination and change the ' + - 'storage class if the storage class header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource, - StorageClass: 'REDUCED_REDUNDANCY' })); - const res = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); - assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); - }); + it( + 'should copy an object to a different destination and change ' + + 'the storage class if storage class header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + }, + ); + + it( + 'should copy an object to the same destination and change the ' + + 'storage class if the storage class header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + CopySource: copySource, + StorageClass: 'REDUCED_REDUNDANCY', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); + assert.strictEqual(res.StorageClass, 'REDUCED_REDUNDANCY'); + }, + ); } - it('should copy an object to a new bucket and overwrite an already ' + - 'existing object in the destination bucket', async () => { - await s3.send(new PutObjectCommand({ Bucket: destBucketName, Key: destObjName, - Body: 'overwrite me', Metadata: originalMetadata })); - const copyRes = await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'REPLACE', - Metadata: newMetadata })); - assert.strictEqual(copyRes.CopyObjectResult.ETag, etag); - const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); - assert.deepStrictEqual(getRes.Metadata, newMetadata); - assert.strictEqual(getRes.ETag, etag); - const body = await getRes.Body.transformToString(); - assert.strictEqual(body, content); - }); + it( + 'should copy an object to a new bucket and overwrite an already ' + + 'existing object in the destination bucket', + async () => { + await s3.send( + new PutObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + Body: 'overwrite me', + Metadata: originalMetadata, + }), + ); + const copyRes = await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'REPLACE', + Metadata: newMetadata, + }), + ); + assert.strictEqual(copyRes.CopyObjectResult.ETag, etag); + const getRes = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.deepStrictEqual(getRes.Metadata, newMetadata); + assert.strictEqual(getRes.ETag, etag); + const body = await getRes.Body.transformToString(); + assert.strictEqual(body, content); + }, + ); // skipping test as object level encryption is not implemented yet - it.skip('should copy an object and change the server side encryption' + - 'option if server side encryption header provided', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - ServerSideEncryption: 'AES256' })); - const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.ServerSideEncryption, 'AES256'); - }); - - it('should return Not Implemented error for obj. encryption using '+ - 'customer-provided encryption keys', async () => { - const params = { Bucket: destBucketName, Key: 'key', - CopySource: copySource, - SSECustomerAlgorithm: 'AES256' }; - try { - await s3.send(new CopyObjectCommand(params)); - assert.fail('Expected NotImplemented error'); - } catch (err) { - assert.strictEqual(err.name, 'NotImplemented'); - } - }); + it.skip( + 'should copy an object and change the server side encryption' + + 'option if server side encryption header provided', + async () => { + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + ServerSideEncryption: 'AES256', + }), + ); + const res = await s3.send(new GetObjectCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.ServerSideEncryption, 'AES256'); + }, + ); + + it( + 'should return Not Implemented error for obj. encryption using ' + 'customer-provided encryption keys', + async () => { + const params = { + Bucket: destBucketName, + Key: 'key', + CopySource: copySource, + SSECustomerAlgorithm: 'AES256', + }; + try { + await s3.send(new CopyObjectCommand(params)); + assert.fail('Expected NotImplemented error'); + } catch (err) { + assert.strictEqual(err.name, 'NotImplemented'); + } + }, + ); it('should copy an object and set the acl on the new object', async () => { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - ACL: 'authenticated-read' })); - const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, - Key: destObjName })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + ACL: 'authenticated-read', + }), + ); + const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, Key: destObjName })); assert.strictEqual(res.Grants.length, 2); assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); assert.strictEqual(res.Grants[1].Permission, 'READ'); - assert.strictEqual(res.Grants[1].Grantee.URI, - 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'); - }); - - it('should copy an object and default the acl on the new object ' + - 'to private even if the copied object had a ' + - 'different acl', async () => { - await s3.send(new PutObjectAclCommand({ Bucket: sourceBucketName, Key: sourceObjName, - ACL: 'authenticated-read', - VersionId: versionId })); - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource })); - const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, - Key: destObjName })); - assert.strictEqual(res.Grants.length, 1); - assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); - }); - - it('should copy a version to same object name to restore '+ - 'version of object', async () => { - const res = await s3.send(new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, - CopySource: copySource })); + assert.strictEqual(res.Grants[1].Grantee.URI, 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'); + }); + + it( + 'should copy an object and default the acl on the new object ' + + 'to private even if the copied object had a ' + + 'different acl', + async () => { + await s3.send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'authenticated-read', + VersionId: versionId, + }), + ); + await s3.send( + new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, CopySource: copySource }), + ); + const res = await s3.send(new GetObjectAclCommand({ Bucket: destBucketName, Key: destObjName })); + assert.strictEqual(res.Grants.length, 1); + assert.strictEqual(res.Grants[0].Permission, 'FULL_CONTROL'); + }, + ); + + it('should copy a version to same object name to restore ' + 'version of object', async () => { + const res = await s3.send( + new CopyObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName, CopySource: copySource }), + ); await successCopyCheck(null, res, originalMetadata, sourceBucketName, sourceObjName); }); it('should return an error if attempt to copy from nonexistent bucket', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `nobucket453234/${sourceObjName}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `nobucket453234/${sourceObjName}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchBucket'); @@ -505,9 +651,14 @@ describe('Object Version Copy', () => { it('should return an error if use invalid redirect location', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - WebsiteRedirectLocation: 'google.com' })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + WebsiteRedirectLocation: 'google.com', + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'InvalidRedirectLocation'); @@ -516,8 +667,13 @@ describe('Object Version Copy', () => { it('should return an error if attempt to copy to nonexistent bucket', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: 'nobucket453234', Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: 'nobucket453234', + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchBucket'); @@ -526,8 +682,13 @@ describe('Object Version Copy', () => { it('should return an error if attempt to copy nonexistent object', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: `${sourceBucketName}/nokey` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/nokey`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchKey'); @@ -535,12 +696,16 @@ describe('Object Version Copy', () => { }); it('should return NoSuchKey if attempt to copy version with delete marker', async () => { - const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })); + const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); assert.strictEqual(delRes.DeleteMarker, true); try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, CopySource: `${sourceBucketName}/${sourceObjName}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'NoSuchKey'); @@ -548,15 +713,17 @@ describe('Object Version Copy', () => { }); it('should return InvalidRequest if attempt to copy specific version that is a delete marker', async () => { - const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, - Key: sourceObjName })); + const delRes = await s3.send(new DeleteObjectCommand({ Bucket: sourceBucketName, Key: sourceObjName })); assert.strictEqual(delRes.DeleteMarker, true); const deleteMarkerId = delRes.VersionId; try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${sourceBucketName}/${sourceObjName}` + - `?versionId=${deleteMarkerId}` })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${sourceBucketName}/${sourceObjName}` + `?versionId=${deleteMarkerId}`, + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'InvalidRequest'); @@ -565,9 +732,14 @@ describe('Object Version Copy', () => { it('should return an error if send invalid metadata directive header', async () => { try { - await s3.send(new CopyObjectCommand({ Bucket: destBucketName, Key: destObjName, - CopySource: copySource, - MetadataDirective: 'copyHalf' })); + await s3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: copySource, + MetadataDirective: 'copyHalf', + }), + ); assert.fail('Expected error'); } catch (err) { checkError(err, 'InvalidArgument'); @@ -586,63 +758,83 @@ describe('Object Version Copy', () => { await otherAccountBucketUtility.deleteOne(otherAccountBucket); }); - it('should not allow an account without read permission on the ' + - 'source object to copy the object', async () => { - try { - await otherAccountS3.send(new CopyObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: copySource })); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'AccessDenied'); - } - }); - - it('should not allow an account without write permission on the ' + - 'destination bucket to copy the object', async () => { - await otherAccountS3.send(new PutObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - Body: '' })); - try { - await otherAccountS3.send(new CopyObjectCommand({ Bucket: destBucketName, - Key: destObjName, - CopySource: `${otherAccountBucket}/${otherAccountKey}` })); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'AccessDenied'); - } - }); - - it('should allow an account with read permission on the ' + - 'source object and write permission on the destination ' + - 'bucket to copy the object', async () => { - await s3.send(new PutObjectAclCommand({ Bucket: sourceBucketName, - Key: sourceObjName, - ACL: 'public-read', - VersionId: versionId })); - await otherAccountS3.send(new CopyObjectCommand({ Bucket: otherAccountBucket, - Key: otherAccountKey, - CopySource: copySource })); - }); - }); - - it('If-Match: returns no error when ETag match, with double quotes ' + - 'around ETag', async () => { + it( + 'should not allow an account without read permission on the ' + 'source object to copy the object', + async () => { + try { + await otherAccountS3.send( + new CopyObjectCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: copySource, + }), + ); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'AccessDenied'); + } + }, + ); + + it( + 'should not allow an account without write permission on the ' + + 'destination bucket to copy the object', + async () => { + await otherAccountS3.send( + new PutObjectCommand({ Bucket: otherAccountBucket, Key: otherAccountKey, Body: '' }), + ); + try { + await otherAccountS3.send( + new CopyObjectCommand({ + Bucket: destBucketName, + Key: destObjName, + CopySource: `${otherAccountBucket}/${otherAccountKey}`, + }), + ); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'AccessDenied'); + } + }, + ); + + it( + 'should allow an account with read permission on the ' + + 'source object and write permission on the destination ' + + 'bucket to copy the object', + async () => { + await s3.send( + new PutObjectAclCommand({ + Bucket: sourceBucketName, + Key: sourceObjName, + ACL: 'public-read', + VersionId: versionId, + }), + ); + await otherAccountS3.send( + new CopyObjectCommand({ + Bucket: otherAccountBucket, + Key: otherAccountKey, + CopySource: copySource, + }), + ); + }, + ); + }); + + it('If-Match: returns no error when ETag match, with double quotes ' + 'around ETag', async () => { await requestCopy({ CopySourceIfMatch: etag }); }); - it('If-Match: returns no error when one of ETags match, with double ' + - 'quotes around ETag', async () => { + it('If-Match: returns no error when one of ETags match, with double ' + 'quotes around ETag', async () => { await requestCopy({ CopySourceIfMatch: `non-matching,${etag}` }); }); - it('If-Match: returns no error when ETag match, without double ' + - 'quotes around ETag', async () => { + it('If-Match: returns no error when ETag match, without double ' + 'quotes around ETag', async () => { await requestCopy({ CopySourceIfMatch: etagTrim }); }); - it('If-Match: returns no error when one of ETags match, without ' + - 'double quotes around ETag', async () => { + it('If-Match: returns no error when one of ETags match, without ' + 'double quotes around ETag', async () => { await requestCopy({ CopySourceIfMatch: `non-matching,${etagTrim}` }); }); @@ -667,8 +859,7 @@ describe('Object Version Copy', () => { await requestCopy({ CopySourceIfNoneMatch: 'non-matching,non-matching-either' }); }); - it('If-None-Match: returns NotModified when ETag match, with double ' + - 'quotes around ETag', async () => { + it('If-None-Match: returns NotModified when ETag match, with double ' + 'quotes around ETag', async () => { try { await requestCopy({ CopySourceIfNoneMatch: etag }); assert.fail('Expected error'); @@ -677,18 +868,19 @@ describe('Object Version Copy', () => { } }); - it('If-None-Match: returns NotModified when one of ETags match, with ' + - 'double quotes around ETag', async () => { - try { - await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etag}` }); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'PreconditionFailed'); - } - }); + it( + 'If-None-Match: returns NotModified when one of ETags match, with ' + 'double quotes around ETag', + async () => { + try { + await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etag}` }); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'PreconditionFailed'); + } + }, + ); - it('If-None-Match: returns NotModified when ETag match, without ' + - 'double quotes around ETag', async () => { + it('If-None-Match: returns NotModified when ETag match, without ' + 'double quotes around ETag', async () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim }); assert.fail('Expected error'); @@ -697,24 +889,24 @@ describe('Object Version Copy', () => { } }); - it('If-None-Match: returns NotModified when one of ETags match, ' + - 'without double quotes around ETag', async () => { - try { - await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etagTrim}` }); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'PreconditionFailed'); - } - }); + it( + 'If-None-Match: returns NotModified when one of ETags match, ' + 'without double quotes around ETag', + async () => { + try { + await requestCopy({ CopySourceIfNoneMatch: `non-matching,${etagTrim}` }); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'PreconditionFailed'); + } + }, + ); - it('If-Modified-Since: returns no error if Last modified date is ' + - 'greater', async () => { + it('If-Modified-Since: returns no error if Last modified date is ' + 'greater', async () => { await requestCopy({ CopySourceIfModifiedSince: dateFromNow(-1) }); }); // Skipping this test, because real AWS does not provide error as // expected - it.skip('If-Modified-Since: returns NotModified if Last modified ' + - 'date is lesser', async () => { + it.skip('If-Modified-Since: returns NotModified if Last modified ' + 'date is lesser', async () => { try { await requestCopy({ CopySourceIfModifiedSince: dateFromNow(1) }); assert.fail('Expected error'); @@ -723,8 +915,7 @@ describe('Object Version Copy', () => { } }); - it('If-Modified-Since: returns NotModified if Last modified '+ - 'date is equal', async () => { + it('If-Modified-Since: returns NotModified if Last modified ' + 'date is equal', async () => { try { await requestCopy({ CopySourceIfModifiedSince: dateConvert(lastModified) }); assert.fail('Expected error'); @@ -733,18 +924,15 @@ describe('Object Version Copy', () => { } }); - it('If-Unmodified-Since: returns no error when lastModified date is ' + - 'greater', async () => { + it('If-Unmodified-Since: returns no error when lastModified date is ' + 'greater', async () => { await requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(1) }); }); - it('If-Unmodified-Since: returns no error when lastModified ' + - 'date is equal', async () => { + it('If-Unmodified-Since: returns no error when lastModified ' + 'date is equal', async () => { await requestCopy({ CopySourceIfUnmodifiedSince: dateConvert(lastModified) }); }); - it('If-Unmodified-Since: returns PreconditionFailed when ' + - 'lastModified date is lesser', async () => { + it('If-Unmodified-Since: returns PreconditionFailed when ' + 'lastModified date is lesser', async () => { try { await requestCopy({ CopySourceIfUnmodifiedSince: dateFromNow(-1) }); assert.fail('Expected error'); @@ -753,10 +941,12 @@ describe('Object Version Copy', () => { } }); - it('If-Match & If-Unmodified-Since: returns no error when match Etag ' + - 'and lastModified is greater', async () => { - await requestCopy({ CopySourceIfMatch: etagTrim, CopySourceIfUnmodifiedSince: dateFromNow(-1) }); - }); + it( + 'If-Match & If-Unmodified-Since: returns no error when match Etag ' + 'and lastModified is greater', + async () => { + await requestCopy({ CopySourceIfMatch: etagTrim, CopySourceIfUnmodifiedSince: dateFromNow(-1) }); + }, + ); it('If-Match match & If-Unmodified-Since match', async () => { await requestCopy({ CopySourceIfMatch: etagTrim, CopySourceIfUnmodifiedSince: dateFromNow(1) }); @@ -775,7 +965,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(1) }); + CopySourceIfUnmodifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -791,14 +982,16 @@ describe('Object Version Copy', () => { it('If-Match match & If-Modified-Since match', async () => { await requestCopy({ CopySourceIfMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(-1) }); + CopySourceIfModifiedSince: dateFromNow(-1), + }); }); it('If-Match not match & If-Modified-Since not match', async () => { try { await requestCopy({ CopySourceIfMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(1) }); + CopySourceIfModifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -809,30 +1002,36 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(-1) }); + CopySourceIfModifiedSince: dateFromNow(-1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); } }); - it('If-None-Match & If-Modified-Since: returns NotModified when Etag ' + - 'does not match and lastModified is greater', async () => { - try { - await requestCopy({ - CopySourceIfNoneMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(-1) }); - assert.fail('Expected error'); - } catch (err) { - checkError(err, 'PreconditionFailed'); - } - }); + it( + 'If-None-Match & If-Modified-Since: returns NotModified when Etag ' + + 'does not match and lastModified is greater', + async () => { + try { + await requestCopy({ + CopySourceIfNoneMatch: etagTrim, + CopySourceIfModifiedSince: dateFromNow(-1), + }); + assert.fail('Expected error'); + } catch (err) { + checkError(err, 'PreconditionFailed'); + } + }, + ); it('If-None-Match not match & If-Modified-Since not match', async () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim, - CopySourceIfModifiedSince: dateFromNow(1) }); + CopySourceIfModifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -842,7 +1041,8 @@ describe('Object Version Copy', () => { it('If-None-Match match & If-Modified-Since match', async () => { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(-1) }); + CopySourceIfModifiedSince: dateFromNow(-1), + }); }); // Skipping this test, because real AWS does not provide error as @@ -851,7 +1051,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfModifiedSince: dateFromNow(1) }); + CopySourceIfModifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -861,14 +1062,16 @@ describe('Object Version Copy', () => { it('If-None-Match match & If-Unmodified-Since match', async () => { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(1) }); + CopySourceIfUnmodifiedSince: dateFromNow(1), + }); }); it('If-None-Match match & If-Unmodified-Since not match', async () => { try { await requestCopy({ CopySourceIfNoneMatch: 'non-matching', - CopySourceIfUnmodifiedSince: dateFromNow(-1) }); + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -879,7 +1082,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(1) }); + CopySourceIfUnmodifiedSince: dateFromNow(1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); @@ -890,7 +1094,8 @@ describe('Object Version Copy', () => { try { await requestCopy({ CopySourceIfNoneMatch: etagTrim, - CopySourceIfUnmodifiedSince: dateFromNow(-1) }); + CopySourceIfUnmodifiedSince: dateFromNow(-1), + }); assert.fail('Expected error'); } catch (err) { checkError(err, 'PreconditionFailed'); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectDelete.js b/tests/functional/aws-node-sdk/test/versioning/objectDelete.js index 34c00968bf..869c99b800 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectDelete.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectDelete.js @@ -13,11 +13,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - versioningSuspended, - versioningEnabled, - removeAllVersions, -} = require('../../lib/utility/versioning-util.js'); +const { versioningSuspended, versioningEnabled, removeAllVersions } = require('../../lib/utility/versioning-util.js'); const { promisify } = require('util'); const removeAllVersionsPromise = promisify(removeAllVersions); @@ -26,9 +22,9 @@ const bucket = `versioning-bucket-${Date.now()}`; const key = 'anObject'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; describe('delete marker creation in bucket with null version', () => { withV4(sigCfg => { @@ -38,11 +34,13 @@ describe('delete marker creation in bucket with null version', () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: nullVersionBody, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: nullVersionBody, + }), + ); }); afterEach(async () => { @@ -58,10 +56,12 @@ describe('delete marker creation in bucket with null version', () => { }); it('should keep the null version if versioning enabled', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); const listData = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket })); assert.strictEqual(listData.Versions.length, 1); @@ -77,12 +77,13 @@ describe('delete marker creation in bucket with null version', () => { assert.strictEqual(listData2.DeleteMarkers[0].VersionId, deleteData.VersionId); }); - it('delete marker overwrites null version if versioning suspended', - async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + it('delete marker overwrites null version if versioning suspended', async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); const listData = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket })); assert.strictEqual(listData.Versions.length, 1); @@ -124,36 +125,42 @@ describe('aws-node-sdk test delete object', () => { } }); - it('delete non existent object should not create a delete marker', - async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}000`, - })); + it('delete non existent object should not create a delete marker', async () => { + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}000`, + }), + ); assert.strictEqual(res.DeleteMarker, undefined); assert.strictEqual(res.VersionId, undefined); }); it('creating non-versioned object', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.equal(res.VersionId, undefined); }); - it('delete in non-versioned bucket should not create delete marker', - async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - })); + it('delete in non-versioned bucket should not create delete marker', async () => { + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.equal(putRes.VersionId, undefined); - const deleteRes = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const deleteRes = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(deleteRes.DeleteMarker, undefined); assert.strictEqual(deleteRes.VersionId, undefined); }); @@ -168,50 +175,60 @@ describe('aws-node-sdk test delete object', () => { await s3.send(new PutBucketVersioningCommand(params)); }); - it('should not send back error for non-existing key (specific version)', - async () => { - await s3.send(new DeleteObjectCommand({ + it('should not send back error for non-existing key (specific version)', async () => { + await s3.send( + new DeleteObjectCommand({ Bucket: bucket, Key: `${key}3`, VersionId: 'null', - })); - }); + }), + ); + }); it('delete non existent object should create a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.notEqual(res.VersionId, undefined); - const res2 = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res2 = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res2.DeleteMarker, true); assert.notEqual(res2.VersionId, res.VersionId); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - VersionId: res.VersionId, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + VersionId: res.VersionId, + }), + ); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - VersionId: res2.VersionId, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + VersionId: res2.VersionId, + }), + ); }); - it('delete non existent version should not create delete marker', - async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: nonExistingId, - })); + it('delete non existent version should not create delete marker', async () => { + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: nonExistingId, + }), + ); assert.strictEqual(res.VersionId, nonExistingId); const listRes = await s3.send(new ListObjectVersionsCommand({ Bucket: bucket })); @@ -219,69 +236,84 @@ describe('aws-node-sdk test delete object', () => { }); it('put a version to the object', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test', + }), + ); versionIds.push('null'); versionIds.push(res.VersionId); assert.notEqual(res.VersionId, undefined); }); it('should create a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.strictEqual( versionIds.find(item => item === res.VersionId), - undefined); + undefined, + ); versionIds.push(res.VersionId); }); it('should return 404 with a delete marker', done => { - s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })).then(() => { - done(new Error('should return 404')); - }).catch(err => { - assert.strictEqual(err.Code, 'NoSuchKey'); - done(); - }); + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ) + .then(() => { + done(new Error('should return 404')); + }) + .catch(err => { + assert.strictEqual(err.Code, 'NoSuchKey'); + done(); + }); }); it('should delete the null version', async () => { const version = versionIds.shift(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.equal(res.DeleteMarker, undefined); }); it('should delete the versioned object', async () => { const version = versionIds.shift(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.equal(res.DeleteMarker, undefined); }); it('should delete the delete-marker version', async () => { const version = versionIds.shift(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.equal(res.DeleteMarker, true); // In AWS SDK v3, the delete marker flag is sufficient for validation @@ -289,22 +321,26 @@ describe('aws-node-sdk test delete object', () => { }); it('put a new version', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test', + }), + ); versionIds.push(res.VersionId); assert.notEqual(res.VersionId, undefined); }); it('get the null version', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); throw new Error('should send back an error'); } catch (err) { if (err.Code !== 'NoSuchVersion') { @@ -324,32 +360,40 @@ describe('aws-node-sdk test delete object', () => { }); it('delete non existent object should create a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.notEqual(res.VersionId, undefined); - const res2 = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - })); + const res2 = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + }), + ); assert.strictEqual(res2.DeleteMarker, true); assert.strictEqual(res2.VersionId, res.VersionId); - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: `${key}2`, - VersionId: res.VersionId, - })); + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: `${key}2`, + VersionId: res.VersionId, + }), + ); }); it('should put a new delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.strictEqual(res.VersionId, 'null'); }); @@ -365,27 +409,33 @@ describe('aws-node-sdk test delete object', () => { }); it('should get the null version', done => { - s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })).then(() => { - done('should return an error'); - }).catch(err => { - if (err.Code !== 'MethodNotAllowed') { - return done(err); - } else { - return done(); - } - }); + s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ) + .then(() => { + done('should return an error'); + }) + .catch(err => { + if (err.Code !== 'MethodNotAllowed') { + return done(err); + } else { + return done(); + } + }); }); it('put a new version to store the null version', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test', + }), + ); versionIds.push(res.VersionId); }); @@ -400,11 +450,13 @@ describe('aws-node-sdk test delete object', () => { }); it('put null version', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'test-null-version', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'test-null-version', + }), + ); assert.strictEqual(res.VersionId, undefined); }); @@ -419,95 +471,115 @@ describe('aws-node-sdk test delete object', () => { }); it('should get the null version', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); const body = await res.Body.transformToString(); assert.strictEqual(body, 'test-null-version'); }); it('should add a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); versionIds.push(res.VersionId); }); it('should get the null version', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); const body = await res.Body.transformToString(); assert.strictEqual(body, 'test-null-version'); }); it('should add a delete marker', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.DeleteMarker, true); assert.strictEqual( versionIds.find(item => item === res.VersionId), - undefined); + undefined, + ); versionIds.push(res.VersionId); }); it('should set the null version as master', async () => { let version = versionIds.pop(); - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res.VersionId, version); assert.strictEqual(res.DeleteMarker, true); - + version = versionIds.pop(); - const res2 = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: version, - })); + const res2 = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: version, + }), + ); assert.strictEqual(res2.VersionId, version); assert.strictEqual(res2.DeleteMarker, true); - - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); + + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); const body = await getRes.Body.transformToString(); assert.strictEqual(body, 'test-null-version'); }); it('should delete null version', async () => { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(res.VersionId, 'null'); - - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); - assert.strictEqual(getRes.VersionId, - versionIds[versionIds.length - 1]); + + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + assert.strictEqual(getRes.VersionId, versionIds[versionIds.length - 1]); }); it('should be able to delete the bucket', async () => { for (const id of versionIds) { - const res = await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: id, - })); + const res = await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: id, + }), + ); assert.strictEqual(res.VersionId, id); } await s3.send(new DeleteBucketCommand({ Bucket: bucket })); @@ -535,11 +607,13 @@ describe('aws-node-sdk test concurrent version-specific deletes with null', () = }); it('creating non-versioned object', async () => { - const res = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: 'null-body', - })); + const res = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: 'null-body', + }), + ); assert.equal(res.VersionId, undefined); }); @@ -556,11 +630,15 @@ describe('aws-node-sdk test concurrent version-specific deletes with null', () = it('put 5 new versions to the object', async () => { const promises = []; for (let i = 0; i < 5; i++) { - promises.push(s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: `test-body-${i}`, - }))); + promises.push( + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: `test-body-${i}`, + }), + ), + ); } await Promise.all(promises); }); @@ -570,16 +648,18 @@ describe('aws-node-sdk test concurrent version-specific deletes with null', () = assert.strictEqual(res.DeleteMarkers, undefined); assert.strictEqual(res.Versions.length, 6); assert.strictEqual(res.Versions[5].VersionId, 'null'); - - await s3.send(new DeleteObjectsCommand({ - Bucket: bucket, - Delete: { - Objects: res.Versions.slice(0, 5).map(item => ({ - Key: item.Key, - VersionId: item.VersionId, - })), - }, - })); + + await s3.send( + new DeleteObjectsCommand({ + Bucket: bucket, + Delete: { + Objects: res.Versions.slice(0, 5).map(item => ({ + Key: item.Key, + VersionId: item.VersionId, + })), + }, + }), + ); }); it('list versions should return a list with just the null version', async () => { diff --git a/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js b/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js index 70d039dc55..07a8b3d4bd 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectDeleteTagging.js @@ -18,10 +18,7 @@ const objectName = 'testtaggingobject'; const invalidId = 'invalidIdWithMoreThan40BytesAndThatIsNotLongEnoughYet'; -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); function _checkError(err, code, statusCode) { assert(err, 'Expected error but found none'); @@ -29,16 +26,15 @@ function _checkError(err, code, statusCode) { assert.strictEqual(err.$metadata?.httpStatusCode, statusCode); } - describe('Delete object tagging with versioning', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); const s3 = bucketUtil.s3; - + beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucketName })); }); - + afterEach(async () => { await removeAllVersions({ Bucket: bucketName }); await bucketUtil.empty(bucketName); @@ -46,172 +42,227 @@ describe('Delete object tagging with versioning', () => { }); it('should be able to delete tag set with versioning', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - const putObjectResult = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + const putObjectResult = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); const versionId = putObjectResult.VersionId; - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { - TagSet: [{ - Key: 'key1', - Value: 'value1', - }] - }, - })); - - const deleteResult = await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })); + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ); + + const deleteResult = await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ); assert.strictEqual(deleteResult.VersionId, versionId); }); - it('should not create version deleting object tags on a ' + - ' version-enabled bucket where no version id is specified ', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - const putObjectResult = await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - const versionId = putObjectResult.VersionId; - - await s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { - TagSet: [{ - Key: 'key1', - Value: 'value1', - }] - }, - })); - - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })); - - await checkOneVersion(s3, bucketName, versionId); - }); - - it('should be able to delete tag set with a version of id "null"', - async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + it( + 'should not create version deleting object tags on a ' + + ' version-enabled bucket where no version id is specified ', + async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + const putObjectResult = await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const versionId = putObjectResult.VersionId; + + await s3.send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ); + + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + await checkOneVersion(s3, bucketName, versionId); + }, + ); + + it('should be able to delete tag set with a version of id "null"', async () => { + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); - const deleteResult = await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: 'null', - })); + const deleteResult = await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: 'null', + }), + ); assert.strictEqual(deleteResult.VersionId, 'null'); }); - it('should return InvalidArgument deleting tag set with a non ' + - 'existing version id', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - try { - await s3.send(new DeleteObjectTaggingCommand({ + it('should return InvalidArgument deleting tag set with a non ' + 'existing version id', async () => { + await s3.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectName, - VersionId: invalidId, - })); - assert.fail('Expected InvalidArgument error'); - } catch (err) { - _checkError(err, 'InvalidArgument', 400); - } - }); - - it('should return 405 MethodNotAllowed deleting tag set without ' + - 'version id if version specified is a delete marker', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); + }), + ); - await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - - try { - await s3.send(new DeleteObjectTaggingCommand({ + await s3.send( + new PutBucketVersioningCommand({ Bucket: bucketName, - Key: objectName, - })); - assert.fail('Expected MethodNotAllowed error'); - } catch (err) { - _checkError(err, 'MethodNotAllowed', 405); - } - }); - - it('should return 405 MethodNotAllowed deleting tag set with ' + - 'version id if version specified is a delete marker', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled - })); - - await s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - - const deleteResult = await s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })); - const versionId = deleteResult.VersionId; + VersioningConfiguration: versioningEnabled, + }), + ); try { - await s3.send(new DeleteObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })); - assert.fail('Expected MethodNotAllowed error'); + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: invalidId, + }), + ); + assert.fail('Expected InvalidArgument error'); } catch (err) { - _checkError(err, 'MethodNotAllowed', 405); + _checkError(err, 'InvalidArgument', 400); } }); + + it( + 'should return 405 MethodNotAllowed deleting tag set without ' + + 'version id if version specified is a delete marker', + async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + try { + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + assert.fail('Expected MethodNotAllowed error'); + } catch (err) { + _checkError(err, 'MethodNotAllowed', 405); + } + }, + ); + + it( + 'should return 405 MethodNotAllowed deleting tag set with ' + + 'version id if version specified is a delete marker', + async () => { + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ); + + await s3.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + + const deleteResult = await s3.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ); + const versionId = deleteResult.VersionId; + + try { + await s3.send( + new DeleteObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ); + assert.fail('Expected MethodNotAllowed error'); + } catch (err) { + _checkError(err, 'MethodNotAllowed', 405); + } + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGet.js b/tests/functional/aws-node-sdk/test/versioning/objectGet.js index 32eb7f4805..4b702e3838 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectGet.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectGet.js @@ -3,35 +3,30 @@ const assert = require('assert'); const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); -const { CreateBucketCommand, + CreateBucketCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutObjectCommand, GetObjectCommand, DeleteObjectCommand, - PutObjectTaggingCommand - } = require('@aws-sdk/client-s3'); + PutObjectTaggingCommand, +} = require('@aws-sdk/client-s3'); const key = 'objectKey'; // formats differ for AWS and S3, use respective sample ids to obtain // correct error response in tests -const nonExistingId = process.env.AWS_ON_AIR ? - 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' : - '3939393939393939393936493939393939393939756e6437'; +const nonExistingId = process.env.AWS_ON_AIR + ? 'MhhyTHhmZ4cxSi4Y9SMe5P7UJAz7HLJ9' + : '3939393939393939393936493939393939393939756e6437'; function _assertError(err, statusCode, code) { - assert.notEqual(err, null, - 'Expected failure but got success'); + assert.notEqual(err, null, 'Expected failure but got success'); assert.strictEqual(err.name, code); assert.strictEqual(err.$metadata.httpStatusCode, statusCode); } - describe('get behavior on versioning-enabled bucket', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -42,10 +37,12 @@ describe('get behavior on versioning-enabled bucket', () => { beforeEach(async () => { bucket = `versioning-bucket-${Date.now()}`; await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(async () => { @@ -61,21 +58,25 @@ describe('get behavior on versioning-enabled bucket', () => { }); it('should be able to get the object version', async () => { - const data = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - })); + const data = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + }), + ); assert.strictEqual(data.ContentLength, 0); }); it('it should return NoSuchVersion if try to get a non-existing object version', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: nonExistingId, - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: nonExistingId, + }), + ); assert.fail('Expected NoSuchVersion error but got success'); } catch (err) { _assertError(err, 404, 'NoSuchVersion'); @@ -84,11 +85,13 @@ describe('get behavior on versioning-enabled bucket', () => { it('it should return NoSuchVersion if try to get a non-existing null version', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.fail('Expected NoSuchVersion error but got success'); } catch (err) { _assertError(err, 404, 'NoSuchVersion'); @@ -96,24 +99,30 @@ describe('get behavior on versioning-enabled bucket', () => { }); it('it should return NoSuchVersion if try to get a deleted noncurrent null version', async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key, VersionId: 'null' })); - + try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.fail('Expected NoSuchVersion error but got success'); } catch (err) { _assertError(err, 404, 'NoSuchVersion'); @@ -123,7 +132,7 @@ describe('get behavior on versioning-enabled bucket', () => { describe('behavior when only version put is a delete marker', () => { let deleteVersionId; - + beforeEach(async () => { const deleteResult = await s3.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })); deleteVersionId = deleteResult.VersionId; @@ -131,11 +140,13 @@ describe('get behavior on versioning-enabled bucket', () => { it('should not be able to get a delete marker', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: deleteVersionId, - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: deleteVersionId, + }), + ); assert.fail('Expected MethodNotAllowed error but got success'); } catch (err) { _assertError(err, 405, 'MethodNotAllowed'); @@ -145,25 +156,28 @@ describe('get behavior on versioning-enabled bucket', () => { } }); - it('it should return NoSuchKey if try to get object whose ' + - 'latest version is a delete marker', async () => { - try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); - assert.fail('Expected NoSuchKey error but got success'); - } catch (err) { - _assertError(err, 404, 'NoSuchKey'); - } - }); + it( + 'it should return NoSuchKey if try to get object whose ' + 'latest version is a delete marker', + async () => { + try { + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + assert.fail('Expected NoSuchKey error but got success'); + } catch (err) { + _assertError(err, 404, 'NoSuchKey'); + } + }, + ); }); - describe('behavior when put version with content then put delete ' + - 'marker', () => { + describe('behavior when put version with content then put delete ' + 'marker', () => { let putVersionId; let deleteVersionId; - + beforeEach(async () => { const putResult = await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key })); putVersionId = putResult.VersionId; @@ -173,47 +187,53 @@ describe('get behavior on versioning-enabled bucket', () => { it('should not be able to get a delete marker', async () => { try { - await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: deleteVersionId, - })); + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: deleteVersionId, + }), + ); assert.fail('Expected MethodNotAllowed error but got success'); } catch (err) { _assertError(err, 405, 'MethodNotAllowed'); } }); - it('should be able to get a version that was put prior to the ' + - 'delete marker', async () => { - const data = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: putVersionId - })); - assert.strictEqual(data.VersionId, putVersionId); - }); - - it('should return NoSuchKey if get object without version and ' + - 'latest version is a delete marker', - async () => { - try { - await s3.send(new GetObjectCommand({ + it('should be able to get a version that was put prior to the ' + 'delete marker', async () => { + const data = await s3.send( + new GetObjectCommand({ Bucket: bucket, Key: key, - })); - assert.fail('Expected NoSuchKey error but got success'); - } catch (err) { - _assertError(err, 404, 'NoSuchKey'); - } + VersionId: putVersionId, + }), + ); + assert.strictEqual(data.VersionId, putVersionId); }); + + it( + 'should return NoSuchKey if get object without version and ' + 'latest version is a delete marker', + async () => { + try { + await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + assert.fail('Expected NoSuchKey error but got success'); + } catch (err) { + _assertError(err, 404, 'NoSuchKey'); + } + }, + ); }); describe('x-amz-tagging-count with versioning', () => { let params; let paramsTagging; let objectVersionId; - + beforeEach(async () => { params = { Bucket: bucket, @@ -235,9 +255,7 @@ describe('get behavior on versioning-enabled bucket', () => { objectVersionId = data.VersionId; }); - it('should not return "x-amz-tagging-count" if no tag ' + - 'associated with the object', - async () => { + it('should not return "x-amz-tagging-count" if no tag ' + 'associated with the object', async () => { params.VersionId = objectVersionId; const data = await s3.send(new GetObjectCommand(params)); assert.strictEqual(data.TagCount, undefined); @@ -249,15 +267,16 @@ describe('get behavior on versioning-enabled bucket', () => { await s3.send(new PutObjectTaggingCommand(paramsTagging)); }); - it('should return "x-amz-tagging-count" header that provides ' + - 'the count of number of tags associated with the object', - async () => { - params.VersionId = objectVersionId; - const data = await s3.send(new GetObjectCommand(params)); - assert.equal(data.TagCount, 1); - }); + it( + 'should return "x-amz-tagging-count" header that provides ' + + 'the count of number of tags associated with the object', + async () => { + params.VersionId = objectVersionId; + const data = await s3.send(new GetObjectCommand(params)); + assert.equal(data.TagCount, 1); + }, + ); }); }); }); }); - diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js b/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js index bae4e16a1c..1943a47162 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectGetAttributes.js @@ -29,10 +29,12 @@ describe('Test get object attributes with versioning', () => { beforeEach(async () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(done => { @@ -40,28 +42,33 @@ describe('Test get object attributes with versioning', () => { if (err) { return done(err); } - return s3.send(new DeleteBucketCommand({ Bucket: bucket })) + return s3 + .send(new DeleteBucketCommand({ Bucket: bucket })) .then(() => done()) .catch(done); }); }); it('should return NoSuchVersion for non-existent versionId', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); const fakeVersionId = '111111111111111111111111111111111111111175636f7270'; try { - await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - VersionId: fakeVersionId, - ObjectAttributes: ['ETag'], - })); + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + VersionId: fakeVersionId, + ObjectAttributes: ['ETag'], + }), + ); assert.fail('Expected NoSuchVersion error'); } catch (err) { assert.strictEqual(err.name, 'NoSuchVersion'); @@ -73,23 +80,29 @@ describe('Test get object attributes with versioning', () => { }); it('should return MethodNotAllowed for delete marker', async () => { - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); - - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); - try { - await s3.send(new GetObjectAttributesCommand({ + await s3.send( + new DeleteObjectCommand({ Bucket: bucket, Key: key, - ObjectAttributes: ['ETag'], - })); + }), + ); + + try { + await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + }), + ); assert.fail('Expected MethodNotAllowed error'); } catch (err) { assert.strictEqual(err.name, 'MethodNotAllowed'); @@ -98,19 +111,23 @@ describe('Test get object attributes with versioning', () => { }); it('should return attributes for specific version', async () => { - const putResult = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); + const putResult = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); const versionId = putResult.VersionId; - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - VersionId: versionId, - ObjectAttributes: ['ETag', 'ObjectSize'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + VersionId: versionId, + ObjectAttributes: ['ETag', 'ObjectSize'], + }), + ); assert.strictEqual(data.ETag, expectedMD5); assert.strictEqual(data.ObjectSize, body.length); @@ -118,18 +135,22 @@ describe('Test get object attributes with versioning', () => { }); it('should return VersionId for versioned object', async () => { - const putResult = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: body, - })); + const putResult = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: body, + }), + ); const versionId = putResult.VersionId; - const data = await s3.send(new GetObjectAttributesCommand({ - Bucket: bucket, - Key: key, - ObjectAttributes: ['ETag'], - })); + const data = await s3.send( + new GetObjectAttributesCommand({ + Bucket: bucket, + Key: key, + ObjectAttributes: ['ETag'], + }), + ); assert.strictEqual(data.VersionId, versionId); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js b/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js index 28c8559ca8..76c8b366f9 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectGetTagging.js @@ -14,10 +14,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); const removeAllVersionsPromise = promisify(removeAllVersions); const bucketName = 'testtaggingbucket'; @@ -45,145 +42,275 @@ describe('Get object tagging with versioning', () => { }); it('should be able to get tag with versioning', done => { - const taggingConfig = { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }; - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(data => next(null, data.VersionId)).catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: taggingConfig, - })).then(() => next(null, versionId)).catch(next), - - (versionId, next) => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })).then(data => next(null, data, versionId)).catch(next), - ], (err, data, versionId) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, versionId); - assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); - done(); - }); + const taggingConfig = { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }; + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: taggingConfig, + }), + ) + .then(() => next(null, versionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ) + .then(data => next(null, data, versionId)) + .catch(next), + ], + (err, data, versionId) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, versionId); + assert.deepStrictEqual(data.TagSet, taggingConfig.TagSet); + done(); + }, + ); }); it('should be able to get tag with a version of id "null"', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: 'null', - })).then(data => next(null, data)).catch(next), - ], (err, data) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, 'null'); - done(); - }); - }); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), - it('should return InvalidArgument getting tag with a non existing ' + - 'version id', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: invalidId, - })).then(data => next(null, data)).catch(next), - ], err => { - _checkError(err, 'InvalidArgument', 400); - done(); - }); - }); + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), - it('should return 404 NoSuchKey getting tag without ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - })).then(data => next(null, data)).catch(next), - ], err => { - _checkError(err, 'NoSuchKey', 404); - done(); - }); + next => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: 'null', + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, 'null'); + done(); + }, + ); }); - it('should return 405 MethodNotAllowed getting tag with ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })).then(() => next()).catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(() => next()).catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName, - })).then(data => next(null, data.VersionId)).catch(next), - - (versionId, next) => s3.send(new GetObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - })).then(data => next(null, data)).catch(next), - ], err => { - _checkError(err, 'MethodNotAllowed', 405); - done(); - }); + it('should return InvalidArgument getting tag with a non existing ' + 'version id', done => { + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: invalidId, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'InvalidArgument', 400); + done(); + }, + ); }); + + it( + 'should return 404 NoSuchKey getting tag without ' + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'NoSuchKey', 404); + done(); + }, + ); + }, + ); + + it( + 'should return 405 MethodNotAllowed getting tag with ' + + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new GetObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'MethodNotAllowed', 405); + done(); + }, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectHead.js b/tests/functional/aws-node-sdk/test/versioning/objectHead.js index df8e448533..ad99ed6c45 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectHead.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectHead.js @@ -13,11 +13,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const removeAllVersionsPromise = promisify(removeAllVersions); const data = ['foo1', 'foo2']; @@ -48,25 +44,26 @@ describe('put and head object with versioning', function testSuite() { await s3.send(new DeleteBucketCommand({ Bucket: bucket })); }); - it('should put and head a non-versioned object without including ' + - 'version ids in response headers', done => { - const params = { Bucket: bucket, Key: key }; - s3.send(new PutObjectCommand(params)) - .then(data => { - _assertNoError(null, 'putting object'); - assert.strictEqual(data.VersionId, undefined); - return s3.send(new HeadObjectCommand(params)); - }) - .then(data => { - _assertNoError(null, 'heading object'); - assert.strictEqual(data.VersionId, undefined); - done(); - }) - .catch(done); - }); + it( + 'should put and head a non-versioned object without including ' + 'version ids in response headers', + done => { + const params = { Bucket: bucket, Key: key }; + s3.send(new PutObjectCommand(params)) + .then(data => { + _assertNoError(null, 'putting object'); + assert.strictEqual(data.VersionId, undefined); + return s3.send(new HeadObjectCommand(params)); + }) + .then(data => { + _assertNoError(null, 'heading object'); + assert.strictEqual(data.VersionId, undefined); + done(); + }) + .catch(done); + }, + ); - it('version-specific head should still not return version id in ' + - 'response header', done => { + it('version-specific head should still not return version id in ' + 'response header', done => { const params = { Bucket: bucket, Key: key }; s3.send(new PutObjectCommand(params)) .then(data => { @@ -85,10 +82,12 @@ describe('put and head object with versioning', function testSuite() { describe('on a version-enabled bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); it('should create a new version for an object', done => { @@ -101,8 +100,7 @@ describe('put and head object with versioning', function testSuite() { }) .then(data => { _assertNoError(null, 'heading object'); - assert.strictEqual(params.VersionId, data.VersionId, - 'version ids are not equal'); + assert.strictEqual(params.VersionId, data.VersionId, 'version ids are not equal'); done(); }) .catch(done); @@ -113,17 +111,21 @@ describe('put and head object with versioning', function testSuite() { const eTags = []; beforeEach(done => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0] - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ) .then(data => { eTags.push(data.ETag); - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + return s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }) .then(() => done()) .catch(done); @@ -135,8 +137,7 @@ describe('put and head object with versioning', function testSuite() { done(); }); - it('should head null version in versioning enabled bucket', - done => { + it('should head null version in versioning enabled bucket', done => { const paramsNull = { Bucket: bucket, Key: '/', @@ -157,21 +158,24 @@ describe('put and head object with versioning', function testSuite() { .then(data => { newVersion = data.VersionId; eTags.push(data.ETag); - return s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: newVersion - })); + return s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: newVersion, + }), + ); }) .then(data => { - assert.strictEqual(data.VersionId, newVersion, - 'version ids are not equal'); + assert.strictEqual(data.VersionId, newVersion, 'version ids are not equal'); assert.strictEqual(data.ETag, eTags[1]); - return s3.send(new HeadObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null' - })); + return s3.send( + new HeadObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); }) .then(data => { _assertNoError(null, 'heading null version'); @@ -182,8 +186,7 @@ describe('put and head object with versioning', function testSuite() { .catch(done); }); - it('should create new versions but still keep nullVersionId', - done => { + it('should create new versions but still keep nullVersionId', done => { const versionIds = []; const params = { Bucket: bucket, Key: key }; const paramsNull = { @@ -192,29 +195,35 @@ describe('put and head object with versioning', function testSuite() { VersionId: 'null', }; // create new versions - async.timesSeries(counter, (i, next) => { - s3.send(new PutObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - // head the 'null' version - return s3.send(new HeadObjectCommand(paramsNull)); - }) - .then(nullVerData => { - assert.strictEqual(nullVerData.ETag, eTags[0]); - assert.strictEqual(nullVerData.VersionId, 'null'); - next(); - }) - .catch(next); - }, done); + async.timesSeries( + counter, + (i, next) => { + s3.send(new PutObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + // head the 'null' version + return s3.send(new HeadObjectCommand(paramsNull)); + }) + .then(nullVerData => { + assert.strictEqual(nullVerData.ETag, eTags[0]); + assert.strictEqual(nullVerData.VersionId, 'null'); + next(); + }) + .catch(next); + }, + done, + ); }); }); describe('on version-suspended bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); it('should not return version id for new object', done => { @@ -252,61 +261,73 @@ describe('put and head object with versioning', function testSuite() { VersionId: 'null', }; const eTags = []; - async.waterfall([ - callback => s3.send(new PutObjectCommand(params1)) - .then(data => { - _assertNoError(null, 'putting first object'); - assert.strictEqual(data.VersionId, undefined); - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(params)) - .then(data => { - _assertNoError(null, 'heading master version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[0], - 'wrong object data'); - callback(); - }) - .catch(callback), - callback => s3.send(new PutObjectCommand(params2)) - .then(data => { - _assertNoError(null, 'putting second object'); - assert.strictEqual(data.VersionId, undefined); - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(paramsNull)) - .then(data => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[1], - 'wrong object data'); - callback(); - }) - .catch(callback), - ], done); + async.waterfall( + [ + callback => + s3 + .send(new PutObjectCommand(params1)) + .then(data => { + _assertNoError(null, 'putting first object'); + assert.strictEqual(data.VersionId, undefined); + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(params)) + .then(data => { + _assertNoError(null, 'heading master version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[0], 'wrong object data'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(params2)) + .then(data => { + _assertNoError(null, 'putting second object'); + assert.strictEqual(data.VersionId, undefined); + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(data => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[1], 'wrong object data'); + callback(); + }) + .catch(callback), + ], + done, + ); }); }); - describe('on a version-suspended bucket with non-versioned object', - () => { + describe('on a version-suspended bucket with non-versioned object', () => { const eTags = []; beforeEach(done => { - s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0] - })) + s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ) .then(data => { eTags.push(data.ETag); - return s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + return s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }) .then(() => done()) .catch(done); @@ -318,8 +339,7 @@ describe('put and head object with versioning', function testSuite() { done(); }); - it('should head null version in versioning suspended bucket', - done => { + it('should head null version in versioning suspended bucket', done => { const paramsNull = { Bucket: bucket, Key: '/', @@ -333,8 +353,7 @@ describe('put and head object with versioning', function testSuite() { .catch(done); }); - it('should update null version in versioning suspended bucket', - done => { + it('should update null version in versioning suspended bucket', done => { const params = { Bucket: bucket, Key: key }; const putParams = { Bucket: bucket, Key: '/', Body: data[1] }; const paramsNull = { @@ -342,69 +361,90 @@ describe('put and head object with versioning', function testSuite() { Key: '/', VersionId: 'null', }; - async.waterfall([ - callback => s3.send(new HeadObjectCommand(paramsNull)) - .then(data => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(data.VersionId, 'null'); - callback(); - }) - .catch(callback), - callback => s3.send(new PutObjectCommand(putParams)) - .then(data => { - _assertNoError(null, 'putting object'); - assert.strictEqual(data.VersionId, undefined); - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(paramsNull)) - .then(data => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[1], - 'wrong object data'); - callback(); - }) - .catch(callback), - callback => s3.send(new HeadObjectCommand(params)) - .then(data => { - _assertNoError(null, 'heading master version'); - assert.strictEqual(data.VersionId, 'null'); - assert.strictEqual(data.ETag, eTags[1], - 'wrong object data'); - callback(); - }) - .catch(callback), - ], done); + async.waterfall( + [ + callback => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(data => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(data.VersionId, 'null'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(putParams)) + .then(data => { + _assertNoError(null, 'putting object'); + assert.strictEqual(data.VersionId, undefined); + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(data => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[1], 'wrong object data'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new HeadObjectCommand(params)) + .then(data => { + _assertNoError(null, 'heading master version'); + assert.strictEqual(data.VersionId, 'null'); + assert.strictEqual(data.ETag, eTags[1], 'wrong object data'); + callback(); + }) + .catch(callback), + ], + done, + ); }); }); - describe('on versioning suspended then enabled bucket w/ null version', - () => { + describe('on versioning suspended then enabled bucket w/ null version', () => { const eTags = []; beforeEach(done => { const params = { Bucket: bucket, Key: key, Body: data[0] }; - async.waterfall([ - callback => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })) - .then(() => callback()) - .catch(callback), - callback => s3.send(new PutObjectCommand(params)) - .then(data => { - eTags.push(data.ETag); - callback(); - }) - .catch(callback), - callback => s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })) - .then(() => callback()) - .catch(callback), - ], done); + async.waterfall( + [ + callback => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ) + .then(() => callback()) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + eTags.push(data.ETag); + callback(); + }) + .catch(callback), + callback => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => callback()) + .catch(callback), + ], + done, + ); }); afterEach(done => { @@ -413,40 +453,51 @@ describe('put and head object with versioning', function testSuite() { done(); }); - it('should preserve the null version when creating new versions', - done => { + it('should preserve the null version when creating new versions', done => { const params = { Bucket: bucket, Key: key }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null', }; - async.waterfall([ - cb => s3.send(new HeadObjectCommand(paramsNull)) - .then(nullVerData => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(nullVerData.ETag, eTags[0]); - assert.strictEqual(nullVerData.VersionId, 'null'); - cb(); - }) - .catch(cb), - cb => async.timesSeries(counter, (i, next) => - s3.send(new PutObjectCommand(params)) - .then(data => { - _assertNoError(null, `putting object #${i}`); - assert.notEqual(data.VersionId, undefined); - next(); - }) - .catch(next), - err => cb(err)), - cb => s3.send(new HeadObjectCommand(paramsNull)) - .then(nullVerData => { - _assertNoError(null, 'heading null version'); - assert.strictEqual(nullVerData.ETag, eTags[0]); - cb(); - }) - .catch(cb), - ], done); + async.waterfall( + [ + cb => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(nullVerData => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(nullVerData.ETag, eTags[0]); + assert.strictEqual(nullVerData.VersionId, 'null'); + cb(); + }) + .catch(cb), + cb => + async.timesSeries( + counter, + (i, next) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + _assertNoError(null, `putting object #${i}`); + assert.notEqual(data.VersionId, undefined); + next(); + }) + .catch(next), + err => cb(err), + ), + cb => + s3 + .send(new HeadObjectCommand(paramsNull)) + .then(nullVerData => { + _assertNoError(null, 'heading null version'); + assert.strictEqual(nullVerData.ETag, eTags[0]); + cb(); + }) + .catch(cb), + ], + done, + ); }); it('should create a bunch of objects and their versions', done => { @@ -454,23 +505,33 @@ describe('put and head object with versioning', function testSuite() { const keycount = 50; const versioncount = 20; const value = '{"foo":"bar"}'; - async.timesLimit(keycount, 10, (i, next1) => { - const key = `foo${i}`; - const params = { Bucket: bucket, Key: key, Body: value }; - async.timesLimit(versioncount, 10, (j, next2) => - s3.send(new PutObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - vids.push({ Key: key, VersionId: data.VersionId }); - next2(); - }) - .catch(next2), - next1); - }, err => { - assert.strictEqual(err, null); - assert.strictEqual(vids.length, keycount * versioncount); - done(); - }); + async.timesLimit( + keycount, + 10, + (i, next1) => { + const key = `foo${i}`; + const params = { Bucket: bucket, Key: key, Body: value }; + async.timesLimit( + versioncount, + 10, + (j, next2) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + vids.push({ Key: key, VersionId: data.VersionId }); + next2(); + }) + .catch(next2), + next1, + ); + }, + err => { + assert.strictEqual(err, null); + assert.strictEqual(vids.length, keycount * versioncount); + done(); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectPut.js b/tests/functional/aws-node-sdk/test/versioning/objectPut.js index fd61452b3b..f68a12e76b 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectPut.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectPut.js @@ -75,10 +75,12 @@ describe('put and get object with versioning', function testSuite() { it('should put and get a non-versioned object without including version ids in response headers', async () => { const params = { Bucket: bucket, Key: key, Body: Buffer.from('') }; - const putRes = await s3.send(new PutObjectCommand({ - ...params, - Body: Buffer.from(''), - })); + const putRes = await s3.send( + new PutObjectCommand({ + ...params, + Body: Buffer.from(''), + }), + ); assert.strictEqual(putRes.VersionId, undefined); const getRes = await s3.send(new GetObjectCommand(params)); @@ -87,40 +89,47 @@ describe('put and get object with versioning', function testSuite() { it('version-specific get should still not return version id in response header', async () => { const params = { Bucket: bucket, Key: key, Body: Buffer.from('') }; - const putRes = await s3.send(new PutObjectCommand({ - ...params, - Body: Buffer.from(''), - })); + const putRes = await s3.send( + new PutObjectCommand({ + ...params, + Body: Buffer.from(''), + }), + ); assert.strictEqual(putRes.VersionId, undefined); - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(getRes.VersionId, undefined); }); describe('on a version-enabled bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); it('should create a new version for an object', async () => { const params = { Bucket: bucket, Key: key, Body: Buffer.from('') }; const putRes = await s3.send(new PutObjectCommand(params)); - const getRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: putRes.VersionId, - })); + const getRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: putRes.VersionId, + }), + ); - assert.strictEqual(putRes.VersionId, getRes.VersionId, - 'version ids are not equal'); + assert.strictEqual(putRes.VersionId, getRes.VersionId, 'version ids are not equal'); }); it('should create a new version with tag set for an object', async () => { @@ -134,14 +143,15 @@ describe('put and get object with versioning', function testSuite() { const putRes = await s3.send(new PutObjectCommand(putParams)); - const tagRes = await s3.send(new GetObjectTaggingCommand({ - Bucket: bucket, - Key: key, - VersionId: putRes.VersionId, - })); + const tagRes = await s3.send( + new GetObjectTaggingCommand({ + Bucket: bucket, + Key: key, + VersionId: putRes.VersionId, + }), + ); - assert.strictEqual(tagRes.VersionId, putRes.VersionId, - 'version ids are not equal'); + assert.strictEqual(tagRes.VersionId, putRes.VersionId, 'version ids are not equal'); assert.strictEqual(tagRes.TagSet[0].Key, tagKey); assert.strictEqual(tagRes.TagSet[0].Value, tagValue); }); @@ -151,68 +161,82 @@ describe('put and get object with versioning', function testSuite() { const eTags = []; beforeEach(async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0], - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ); eTags.length = 0; eTags.push(putRes.ETag); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(() => { eTags.length = 0; }); - it('should get null (latest) version in versioning enabled ' + - 'bucket when version id is not specified', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); - - assert.strictEqual(res.VersionId, 'null'); - }); - - it('should get null version in versioning enabled bucket ' + - 'when version id is specified', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + it( + 'should get null (latest) version in versioning enabled ' + 'bucket when version id is not specified', + async () => { + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + assert.strictEqual(res.VersionId, 'null'); + }, + ); + + it('should get null version in versioning enabled bucket ' + 'when version id is specified', async () => { + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(res.VersionId, 'null'); }); it('should keep null version and create a new version', async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[1], - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[1], + }), + ); const newVersion = putRes.VersionId; eTags.push(putRes.ETag); - const newVerRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: newVersion, - })); - assert.strictEqual(newVerRes.VersionId, newVersion, - 'version ids are not equal'); + const newVerRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: newVersion, + }), + ); + assert.strictEqual(newVerRes.VersionId, newVersion, 'version ids are not equal'); assert.strictEqual(newVerRes.ETag, eTags[1]); - const nullRes = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const nullRes = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(nullRes.VersionId, 'null'); assert.strictEqual(nullRes.ETag, eTags[0]); }); @@ -236,52 +260,67 @@ describe('put and get object with versioning', function testSuite() { }); // S3C-5139 - it('should not fail PUT on versioning-suspended bucket if nullVersionId refers ' + - 'to deleted null version', async () => { - // create a new version on top of non-versioned object - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - })); - - // suspend versioning - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); - - // delete existing non-versioned object - await s3.send(new DeleteObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); - - // put a new null version - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0], - })); - eTags[0] = putRes.ETag; - - // get the new null version - const nullVerData = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); - assert.strictEqual(nullVerData.ETag, eTags[0]); - assert.strictEqual(nullVerData.VersionId, 'null'); - }); + it( + 'should not fail PUT on versioning-suspended bucket if nullVersionId refers ' + + 'to deleted null version', + async () => { + // create a new version on top of non-versioned object + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); + + // suspend versioning + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); + + // delete existing non-versioned object + await s3.send( + new DeleteObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); + + // put a new null version + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ); + eTags[0] = putRes.ETag; + + // get the new null version + const nullVerData = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); + assert.strictEqual(nullVerData.ETag, eTags[0]); + assert.strictEqual(nullVerData.VersionId, 'null'); + }, + ); }); describe('on version-suspended bucket', () => { beforeEach(async () => { - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); it('should not return version id for new object', async () => { @@ -334,86 +373,103 @@ describe('put and get object with versioning', function testSuite() { }); // Jira issue: S3C-444 - it('put object after put object acl on null version which is ' + - 'latest version should not result in two null version with ' + - 'different version ids', async () => { - // create new null version (master version in metadata) - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); - await checkOneVersion(s3, bucket, 'null'); - - // apply ACL on null version - await s3.send(new PutObjectAclCommand({ - Bucket: bucket, - Key: key, - ACL: 'public-read-write', - VersionId: 'null', - })); - - // before overwriting master version, put object should clean up latest null version - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); - - // if clean-up did not occur, would see two null versions with different version IDs - await checkOneVersion(s3, bucket, 'null'); - }); + it( + 'put object after put object acl on null version which is ' + + 'latest version should not result in two null version with ' + + 'different version ids', + async () => { + // create new null version (master version in metadata) + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); + await checkOneVersion(s3, bucket, 'null'); + + // apply ACL on null version + await s3.send( + new PutObjectAclCommand({ + Bucket: bucket, + Key: key, + ACL: 'public-read-write', + VersionId: 'null', + }), + ); + + // before overwriting master version, put object should clean up latest null version + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); + + // if clean-up did not occur, would see two null versions with different version IDs + await checkOneVersion(s3, bucket, 'null'); + }, + ); // Jira issue: S3C-444 - it('put object after creating dual null version another way ' + - 'should not result in two null version with different version ids', async () => { - // create dual null version state another way - await createDualNullVersionAsync(s3, bucket, key); - - // versioning is left enabled after above step - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); - - // before overwriting master version, put object should clean up latest null version - await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); - - // if clean-up did not occur, would see two null versions with different version IDs - await checkOneVersion(s3, bucket, 'null'); - }); + it( + 'put object after creating dual null version another way ' + + 'should not result in two null version with different version ids', + async () => { + // create dual null version state another way + await createDualNullVersionAsync(s3, bucket, key); + + // versioning is left enabled after above step + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); + + // before overwriting master version, put object should clean up latest null version + await s3.send(new PutObjectCommand({ Bucket: bucket, Key: key, Body: Buffer.from('') })); + + // if clean-up did not occur, would see two null versions with different version IDs + await checkOneVersion(s3, bucket, 'null'); + }, + ); }); describe('on a version-suspended bucket with non-versioned object', () => { const eTags = []; beforeEach(async () => { - const putRes = await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: key, - Body: data[0], - })); + const putRes = await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: key, + Body: data[0], + }), + ); eTags.length = 0; eTags.push(putRes.ETag); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); }); afterEach(() => { eTags.length = 0; }); - it('should get null version (latest) in versioning suspended bucket without specifying version id', - async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - })); + it('should get null version (latest) in versioning suspended bucket without specifying version id', async () => { + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + }), + ); assert.strictEqual(res.VersionId, 'null'); }); it('should get null version in versioning suspended bucket specifying version id', async () => { - const res = await s3.send(new GetObjectCommand({ - Bucket: bucket, - Key: key, - VersionId: 'null', - })); + const res = await s3.send( + new GetObjectCommand({ + Bucket: bucket, + Key: key, + VersionId: 'null', + }), + ); assert.strictEqual(res.VersionId, 'null'); }); @@ -450,19 +506,23 @@ describe('put and get object with versioning', function testSuite() { beforeEach(async () => { const params = { Bucket: bucket, Key: key, Body: data[0] }; - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningSuspended, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningSuspended, + }), + ); const putRes = await s3.send(new PutObjectCommand(params)); eTags.length = 0; eTags.push(putRes.ETag); - await s3.send(new PutBucketVersioningCommand({ - Bucket: bucket, - VersioningConfiguration: versioningEnabled, - })); + await s3.send( + new PutBucketVersioningCommand({ + Bucket: bucket, + VersioningConfiguration: versioningEnabled, + }), + ); }); afterEach(() => { diff --git a/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js b/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js index fa1183b2cf..329bf0f894 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectPutCopyPart.js @@ -17,11 +17,7 @@ const { const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); -const { - removeAllVersions, - versioningEnabled, - versioningSuspended, -} = require('../../lib/utility/versioning-util.js'); +const { removeAllVersions, versioningEnabled, versioningSuspended } = require('../../lib/utility/versioning-util.js'); const removeAllVersionsPromise = promisify(removeAllVersions); let sourceBucket; @@ -30,7 +26,6 @@ const sourceKey = 'sourceobjectkey'; const destKey = 'destobjectkey'; const invalidId = 'invalidIdWithMoreThan40BytesAndThatIsNotLongEnoughYet'; - describe('Object Part Copy with Versioning', () => { withV4(sigCfg => { const bucketUtil = new BucketUtility('default', sigCfg); @@ -40,26 +35,36 @@ describe('Object Part Copy with Versioning', () => { beforeEach(done => { sourceBucket = `copypartsourcebucket-${Date.now()}`; destBucket = `copypartdestbucket-${Date.now()}`; - async.forEach([sourceBucket, destBucket], (bucket, cb) => { - s3.send(new CreateBucketCommand({ Bucket: bucket })) - .then(() => cb()) - .catch(cb); - }, done); + async.forEach( + [sourceBucket, destBucket], + (bucket, cb) => { + s3.send(new CreateBucketCommand({ Bucket: bucket })) + .then(() => cb()) + .catch(cb); + }, + done, + ); }); afterEach(done => { - s3.send(new AbortMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey, - UploadId: uploadId, - })) + s3.send( + new AbortMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + UploadId: uploadId, + }), + ) .then(() => { - async.each([sourceBucket, destBucket], (bucket, cb) => { - removeAllVersionsPromise({ Bucket: bucket }) - .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) - .then(() => cb()) - .catch(cb); - }, done); + async.each( + [sourceBucket, destBucket], + (bucket, cb) => { + removeAllVersionsPromise({ Bucket: bucket }) + .then(() => s3.send(new DeleteBucketCommand({ Bucket: bucket }))) + .then(() => cb()) + .catch(cb); + }, + done, + ); }) .catch(err => { if (err) { @@ -73,30 +78,39 @@ describe('Object Part Copy with Versioning', () => { const eTags = []; beforeEach(done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - Body: 'foobar' - })) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - eTags.push(data.ETag); - s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey - })) - .then(data => next(null, data)) - .catch(next); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + Body: 'foobar', + }), + ) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + eTags.push(data.ETag); + s3.send( + new CreateMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + }), + ) + .then(data => next(null, data)) + .catch(next); + }, + ], + (err, data) => { + if (err) { + return done(err); + } + uploadId = data.UploadId; + return done(); }, - ], (err, data) => { - if (err) { - return done(err); - } - uploadId = data.UploadId; - return done(); - }); + ); }); afterEach(done => { @@ -104,15 +118,16 @@ describe('Object Part Copy with Versioning', () => { done(); }); - it('should not return a version id when put part by copying ' + - 'without specifying version id', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + it('should not return a version id when put part by copying ' + 'without specifying version id', done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, undefined); assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); @@ -121,16 +136,16 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('should return NoSuchKey if copy source version id is invalid ' + - 'id', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?` + - `versionId=${invalidId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + it('should return NoSuchKey if copy source version id is invalid ' + 'id', done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?` + `versionId=${invalidId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(() => { done(new Error('Expected error but got success')); }) @@ -142,22 +157,27 @@ describe('Object Part Copy with Versioning', () => { }); }); - it('should allow specific version "null" for copy source ' + - 'and return version id "null" in response headers', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, 'null'); - assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); - done(); - }) - .catch(done); - }); + it( + 'should allow specific version "null" for copy source ' + + 'and return version id "null" in response headers', + done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, 'null'); + assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); + done(); + }) + .catch(done); + }, + ); }); describe('on bucket with versioning', () => { @@ -167,46 +187,64 @@ describe('Object Part Copy with Versioning', () => { beforeEach(done => { const params = { Bucket: sourceBucket, Key: sourceKey }; - async.waterfall([ - next => s3.send(new PutObjectCommand(params)) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - eTags.push(data.ETag); - versionIds.push('null'); - s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand(params)) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + eTags.push(data.ETag); + versionIds.push('null'); + s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + async.timesSeries( + counter, + (i, cb) => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + Body: `foo${i}`, + }), + ) + .then(data => { + eTags.push(data.ETag); + versionIds.push(data.VersionId); + cb(); + }) + .catch(cb), + err => next(err), + ), + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + if (err) { + return done(err); + } + uploadId = data.UploadId; + return done(); }, - next => async.timesSeries(counter, (i, cb) => - s3.send(new PutObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - Body: `foo${i}` - })) - .then(data => { - eTags.push(data.ETag); - versionIds.push(data.VersionId); - cb(); - }) - .catch(cb), - err => next(err)), - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey - })) - .then(data => next(null, data)) - .catch(next), - ], (err, data) => { - if (err) { - return done(err); - } - uploadId = data.UploadId; - return done(); - }); + ); }); afterEach(done => { @@ -215,102 +253,132 @@ describe('Object Part Copy with Versioning', () => { done(); }); - it('copy part without specifying version should return data and ' + - 'version id of latest version', done => { - const lastVersion = versionIds[versionIds.length - 1]; - const lastETag = eTags[eTags.length - 1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, lastVersion); - assert.strictEqual(data.CopyPartResult.ETag, lastETag); - done(); - }) - .catch(done); - }); - - it('copy part without specifying version should return NoSuchKey ' + - 'if latest version has a delete marker', done => { - s3.send(new DeleteObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey - })) - .then(() => s3.send(new UploadPartCopyCommand({ + it( + 'copy part without specifying version should return data and ' + 'version id of latest version', + done => { + const lastVersion = versionIds[versionIds.length - 1]; + const lastETag = eTags[eTags.length - 1]; + s3.send( + new UploadPartCopyCommand({ Bucket: destBucket, CopySource: `${sourceBucket}/${sourceKey}`, Key: destKey, PartNumber: 1, UploadId: uploadId, - }))) - .then(() => { - done(new Error('Expected err but did not find one')); - }) - .catch(err => { - assert(err, 'Expected err but did not find one'); - assert.strictEqual(err.name, 'NoSuchKey'); - assert.strictEqual(err.$metadata?.httpStatusCode, 404); - done(); - }); - }); + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, lastVersion); + assert.strictEqual(data.CopyPartResult.ETag, lastETag); + done(); + }) + .catch(done); + }, + ); - it('copy part with specific version id should return ' + - 'InvalidRequest if that id is a delete marker', done => { - async.waterfall([ - next => s3.send(new DeleteObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - })) - .then(() => next()) - .catch(next), - next => s3.send(new ListObjectVersionsCommand({ - Bucket: sourceBucket - })) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - const deleteMarkerId = data.DeleteMarkers[0].VersionId; - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${deleteMarkerId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => next(null, data)) - .catch(next); - }, - ], err => { - assert(err, 'Expected err but did not find one'); - assert.strictEqual(err.name, 'InvalidRequest'); - assert.strictEqual(err.$metadata?.httpStatusCode, 400); - done(); - }); - }); + it( + 'copy part without specifying version should return NoSuchKey ' + + 'if latest version has a delete marker', + done => { + s3.send( + new DeleteObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + }), + ) + .then(() => + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ), + ) + .then(() => { + done(new Error('Expected err but did not find one')); + }) + .catch(err => { + assert(err, 'Expected err but did not find one'); + assert.strictEqual(err.name, 'NoSuchKey'); + assert.strictEqual(err.$metadata?.httpStatusCode, 404); + done(); + }); + }, + ); + + it( + 'copy part with specific version id should return ' + 'InvalidRequest if that id is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new ListObjectVersionsCommand({ + Bucket: sourceBucket, + }), + ) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + const deleteMarkerId = data.DeleteMarkers[0].VersionId; + return s3 + .send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${deleteMarkerId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => next(null, data)) + .catch(next); + }, + ], + err => { + assert(err, 'Expected err but did not find one'); + assert.strictEqual(err.name, 'InvalidRequest'); + assert.strictEqual(err.$metadata?.httpStatusCode, 400); + done(); + }, + ); + }, + ); - it('copy part with specific version should return NoSuchVersion ' + - 'if version does not exist', done => { + it('copy part with specific version should return NoSuchVersion ' + 'if version does not exist', done => { const versionId = versionIds[1]; - s3.send(new DeleteObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - VersionId: versionId - })) + s3.send( + new DeleteObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + VersionId: versionId, + }), + ) .then(data => { assert.strictEqual(data.VersionId, versionId); - return s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${versionId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })); + return s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${versionId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ); }) .then(() => { done(new Error('Expected err but did not find one')); @@ -323,17 +391,17 @@ describe('Object Part Copy with Versioning', () => { }); }); - it('copy part with specific version should return copy source ' + - 'version id if it exists', done => { + it('copy part with specific version should return copy source ' + 'version id if it exists', done => { const versionId = versionIds[1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${versionId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${versionId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, versionId); assert.strictEqual(data.CopyPartResult.ETag, eTags[1]); @@ -342,22 +410,26 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('copy part with specific version "null" should return copy ' + - 'source version id "null" if it exists', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, 'null'); - assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); - done(); - }) - .catch(done); - }); + it( + 'copy part with specific version "null" should return copy ' + 'source version id "null" if it exists', + done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, 'null'); + assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); + done(); + }) + .catch(done); + }, + ); }); describe('on bucket with versioning suspended', () => { @@ -367,54 +439,74 @@ describe('Object Part Copy with Versioning', () => { beforeEach(done => { const params = { Bucket: sourceBucket, Key: sourceKey }; - async.waterfall([ - next => s3.send(new PutObjectCommand(params)) - .then(data => next(null, data)) - .catch(next), - (data, next) => { - eTags.push(data.ETag); - versionIds.push('null'); - s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next); - }, - next => async.timesSeries(counter, (i, cb) => - s3.send(new PutObjectCommand({ - Bucket: sourceBucket, - Key: sourceKey, - Body: `foo${i}` - })) - .then(data => { - eTags.push(data.ETag); - versionIds.push(data.VersionId); - cb(); - }) - .catch(cb), - err => next(err)), - next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: sourceBucket, - VersioningConfiguration: versioningSuspended, - })) - .then(() => next()) - .catch(next); + async.waterfall( + [ + next => + s3 + .send(new PutObjectCommand(params)) + .then(data => next(null, data)) + .catch(next), + (data, next) => { + eTags.push(data.ETag); + versionIds.push('null'); + s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + async.timesSeries( + counter, + (i, cb) => + s3 + .send( + new PutObjectCommand({ + Bucket: sourceBucket, + Key: sourceKey, + Body: `foo${i}`, + }), + ) + .then(data => { + eTags.push(data.ETag); + versionIds.push(data.VersionId); + cb(); + }) + .catch(cb), + err => next(err), + ), + next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: sourceBucket, + VersioningConfiguration: versioningSuspended, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + s3 + .send( + new CreateMultipartUploadCommand({ + Bucket: destBucket, + Key: destKey, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + if (err) { + return done(err); + } + uploadId = data.UploadId; + return done(); }, - next => s3.send(new CreateMultipartUploadCommand({ - Bucket: destBucket, - Key: destKey - })) - .then(data => next(null, data)) - .catch(next), - ], (err, data) => { - if (err) { - return done(err); - } - uploadId = data.UploadId; - return done(); - }); + ); }); afterEach(done => { @@ -423,17 +515,18 @@ describe('Object Part Copy with Versioning', () => { done(); }); - it('copy part without specifying version should still return ' + - 'version id of latest version', done => { + it('copy part without specifying version should still return ' + 'version id of latest version', done => { const lastVersion = versionIds[versionIds.length - 1]; const lastETag = eTags[eTags.length - 1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, lastVersion); assert.strictEqual(data.CopyPartResult.ETag, lastETag); @@ -442,17 +535,17 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('copy part with specific version should still return copy ' + - 'source version id if it exists', done => { + it('copy part with specific version should still return copy ' + 'source version id if it exists', done => { const versionId = versionIds[1]; - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}` + - `?versionId=${versionId}`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}` + `?versionId=${versionId}`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) .then(data => { assert.strictEqual(data.CopySourceVersionId, versionId); assert.strictEqual(data.CopyPartResult.ETag, eTags[1]); @@ -461,22 +554,27 @@ describe('Object Part Copy with Versioning', () => { .catch(done); }); - it('copy part with specific version "null" should still return ' + - 'copy source version id "null" if it exists', done => { - s3.send(new UploadPartCopyCommand({ - Bucket: destBucket, - CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, - Key: destKey, - PartNumber: 1, - UploadId: uploadId, - })) - .then(data => { - assert.strictEqual(data.CopySourceVersionId, 'null'); - assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); - done(); - }) - .catch(done); - }); + it( + 'copy part with specific version "null" should still return ' + + 'copy source version id "null" if it exists', + done => { + s3.send( + new UploadPartCopyCommand({ + Bucket: destBucket, + CopySource: `${sourceBucket}/${sourceKey}?versionId=null`, + Key: destKey, + PartNumber: 1, + UploadId: uploadId, + }), + ) + .then(data => { + assert.strictEqual(data.CopySourceVersionId, 'null'); + assert.strictEqual(data.CopyPartResult.ETag, eTags[0]); + done(); + }) + .catch(done); + }, + ); }); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js b/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js index b71919fba8..d2c8fca202 100644 --- a/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js +++ b/tests/functional/aws-node-sdk/test/versioning/objectPutTagging.js @@ -1,6 +1,6 @@ const assert = require('assert'); const async = require('async'); -const {promisify} = require('util'); +const { promisify } = require('util'); const { CreateBucketCommand, @@ -15,12 +15,9 @@ const withV4 = require('../support/withV4'); const BucketUtility = require('../../lib/utility/bucket-util'); const { checkOneVersion } = require('../../lib/utility/versioning-util'); -const { - removeAllVersions, - versioningEnabled, -} = require('../../lib/utility/versioning-util'); +const { removeAllVersions, versioningEnabled } = require('../../lib/utility/versioning-util'); -const removeAllVersionsPromise= promisify(removeAllVersions); +const removeAllVersionsPromise = promisify(removeAllVersions); const bucketName = 'testtaggingbucket'; const objectName = 'testtaggingobject'; @@ -49,227 +46,351 @@ describe('Put object tagging with versioning', () => { }); it('should be able to put tag with versioning', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(data => next(null, data.VersionId)) - .catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data, versionId)) - .catch(next), - ], (err, data, versionId) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, versionId); - done(); - }); - }); + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), - it('should not create version putting object tags on a ' + - ' version-enabled bucket where no version id is specified ', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(data => next(null, data.VersionId)) - .catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(() => next(null, versionId)) - .catch(next), - - (versionId, next) => - checkOneVersion(s3, bucketName, versionId) - .then(() => next()) - .catch(next), - ], done); + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data, versionId)) + .catch(next), + ], + (err, data, versionId) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, versionId); + done(); + }, + ); }); + it( + 'should not create version putting object tags on a ' + + ' version-enabled bucket where no version id is specified ', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(() => next(null, versionId)) + .catch(next), + + (versionId, next) => + checkOneVersion(s3, bucketName, versionId) + .then(() => next()) + .catch(next), + ], + done, + ); + }, + ); + it('should be able to put tag with a version of id "null"', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: 'null', - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], (err, data) => { - assert.ifError(err, `Found unexpected err ${err}`); - assert.strictEqual(data.VersionId, 'null'); - done(); - }); - }); + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), - it('should return InvalidArgument putting tag with a non existing ' + - 'version id', done => { - async.waterfall([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: invalidId, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], err => { - _checkError(err, 'InvalidArgument', 400); - done(); - }); - }); + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), - it('should return 405 MethodNotAllowed putting tag without ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], err => { - _checkError(err, 'MethodNotAllowed', 405); - done(); - }); + next => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: 'null', + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + (err, data) => { + assert.ifError(err, `Found unexpected err ${err}`); + assert.strictEqual(data.VersionId, 'null'); + done(); + }, + ); }); - it('should return 405 MethodNotAllowed putting tag with ' + - 'version id if version specified is a delete marker', done => { - async.waterfall([ - next => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: versioningEnabled, - })) - .then(() => next()) - .catch(next), - - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(() => next()) - .catch(next), - - next => s3.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: objectName - })) - .then(data => next(null, data.VersionId)) - .catch(next), - - (versionId, next) => s3.send(new PutObjectTaggingCommand({ - Bucket: bucketName, - Key: objectName, - VersionId: versionId, - Tagging: { TagSet: [ - { - Key: 'key1', - Value: 'value1', - }] }, - })) - .then(data => next(null, data)) - .catch(next), - ], err => { - _checkError(err, 'MethodNotAllowed', 405); - done(); - }); + it('should return InvalidArgument putting tag with a non existing ' + 'version id', done => { + async.waterfall( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: invalidId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'InvalidArgument', 400); + done(); + }, + ); }); + + it( + 'should return 405 MethodNotAllowed putting tag without ' + + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'MethodNotAllowed', 405); + done(); + }, + ); + }, + ); + + it( + 'should return 405 MethodNotAllowed putting tag with ' + + 'version id if version specified is a delete marker', + done => { + async.waterfall( + [ + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: versioningEnabled, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(() => next()) + .catch(next), + + next => + s3 + .send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: objectName, + }), + ) + .then(data => next(null, data.VersionId)) + .catch(next), + + (versionId, next) => + s3 + .send( + new PutObjectTaggingCommand({ + Bucket: bucketName, + Key: objectName, + VersionId: versionId, + Tagging: { + TagSet: [ + { + Key: 'key1', + Value: 'value1', + }, + ], + }, + }), + ) + .then(data => next(null, data)) + .catch(next), + ], + err => { + _checkError(err, 'MethodNotAllowed', 405); + done(); + }, + ); + }, + ); }); }); diff --git a/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js b/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js index 9bc9af2ad4..492aaa0f8f 100644 --- a/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js +++ b/tests/functional/aws-node-sdk/test/versioning/replicationBucket.js @@ -23,15 +23,14 @@ function checkNoError(err) { } function testVersioning(s3, versioningStatus, replicationStatus, removeReplication, cb) { - const versioningParams = { + const versioningParams = { Bucket: bucketName, - VersioningConfiguration: { Status: versioningStatus } + VersioningConfiguration: { Status: versioningStatus }, }; const replicationParams = { Bucket: bucketName, ReplicationConfiguration: { - Role: 'arn:aws:iam::123456789012:role/examplerole,' + - 'arn:aws:iam::123456789012:role/examplerole', + Role: 'arn:aws:iam::123456789012:role/examplerole,' + 'arn:aws:iam::123456789012:role/examplerole', Rules: [ { Destination: { @@ -44,23 +43,31 @@ function testVersioning(s3, versioningStatus, replicationStatus, removeReplicati ], }, }; - - async.waterfall([ - cb => s3.send(new PutBucketReplicationCommand(replicationParams)) - .then(() => cb()) - .catch(cb), - cb => { - if (removeReplication) { - return s3.send(new DeleteBucketReplicationCommand({ Bucket: bucketName })) + + async.waterfall( + [ + cb => + s3 + .send(new PutBucketReplicationCommand(replicationParams)) .then(() => cb()) - .catch(cb); - } - return process.nextTick(() => cb()); - }, - cb => s3.send(new PutBucketVersioningCommand(versioningParams)) - .then(() => cb()) - .catch(cb), - ], cb); + .catch(cb), + cb => { + if (removeReplication) { + return s3 + .send(new DeleteBucketReplicationCommand({ Bucket: bucketName })) + .then(() => cb()) + .catch(cb); + } + return process.nextTick(() => cb()); + }, + cb => + s3 + .send(new PutBucketVersioningCommand(versioningParams)) + .then(() => cb()) + .catch(cb), + ], + cb, + ); } describe('Versioning on a replication source bucket', () => { @@ -69,35 +76,42 @@ describe('Versioning on a replication source bucket', () => { const s3 = bucketUtil.s3; beforeEach(done => { - async.waterfall([ - cb => s3.send(new CreateBucketCommand({ Bucket: bucketName })) - .then(() => cb()) - .catch(cb), - cb => s3.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { - Status: 'Enabled', - }, - })) - .then(() => cb()) - .catch(cb), - ], done); + async.waterfall( + [ + cb => + s3 + .send(new CreateBucketCommand({ Bucket: bucketName })) + .then(() => cb()) + .catch(cb), + cb => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { + Status: 'Enabled', + }, + }), + ) + .then(() => cb()) + .catch(cb), + ], + done, + ); }); afterEach(async () => { await s3.send(new DeleteBucketCommand({ Bucket: bucketName })); }); - it('should not be able to disable versioning if replication enabled', - done => { + it('should not be able to disable versioning if replication enabled', done => { testVersioning(s3, 'Suspended', 'Enabled', false, err => { checkError(err, 'InvalidBucketState'); done(); }); }); - it('should be able to suspend versioning if replication disabled', - done => { + it('should be able to suspend versioning if replication disabled', done => { testVersioning(s3, 'Suspended', 'Disabled', false, err => { checkNoError(err); done(); diff --git a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js index ecba087ebc..3b1be507af 100644 --- a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js +++ b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral1.js @@ -63,111 +63,134 @@ describe('aws-node-sdk test bucket versioning listing', function testSuite() { const keycount = 20; const versioncount = 20; const value = '{"foo":"bar"}'; - async.timesLimit(keycount, 10, (i, next1) => { - const key = `foo${i}`; - masterVersions.push(key); - const params = { Bucket: bucket, Key: key, Body: value }; - async.timesLimit(versioncount, 10, (j, next2) => - s3.send(new PutObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - allVersions.push({ Key: key, VersionId: data.VersionId }); - next2(); - }) - .catch(next2), - next1); - }, err => { - assert.strictEqual(err, null); - assert.strictEqual(allVersions.length, keycount * versioncount); - done(); - }); + async.timesLimit( + keycount, + 10, + (i, next1) => { + const key = `foo${i}`; + masterVersions.push(key); + const params = { Bucket: bucket, Key: key, Body: value }; + async.timesLimit( + versioncount, + 10, + (j, next2) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + allVersions.push({ Key: key, VersionId: data.VersionId }); + next2(); + }) + .catch(next2), + next1, + ); + }, + err => { + assert.strictEqual(err, null); + assert.strictEqual(allVersions.length, keycount * versioncount); + done(); + }, + ); }); it('should list all latest versions', async () => { const params = { Bucket: bucket, MaxKeys: 1000, Delimiter: '/' }; const data = await s3.send(new ListObjectsCommand(params)); const keys = data.Contents.map(entry => entry.Key); - assert.deepStrictEqual(keys.sort(), masterVersions.sort(), - 'not same keys'); + assert.deepStrictEqual(keys.sort(), masterVersions.sort(), 'not same keys'); }); it('should create some delete markers', done => { const keycount = 15; - async.times(keycount, (i, next) => { - const key = masterVersions[i]; - const params = { Bucket: bucket, Key: key }; - s3.send(new DeleteObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - allVersions.push({ Key: key, VersionId: data.VersionId }); - next(); - }) - .catch(next); - }, done); + async.times( + keycount, + (i, next) => { + const key = masterVersions[i]; + const params = { Bucket: bucket, Key: key }; + s3.send(new DeleteObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + allVersions.push({ Key: key, VersionId: data.VersionId }); + next(); + }) + .catch(next); + }, + done, + ); }); it('should list all latest versions', async () => { const params = { Bucket: bucket, MaxKeys: 1000, Delimiter: '/' }; const data = await s3.send(new ListObjectsCommand(params)); const keys = data.Contents.map(entry => entry.Key); - assert.deepStrictEqual(keys.sort(), masterVersions.sort().slice(15), - 'not same keys'); + assert.deepStrictEqual(keys.sort(), masterVersions.sort().slice(15), 'not same keys'); }); it('should list all versions', done => { const versions = []; const params = { Bucket: bucket, MaxKeys: 15, Delimiter: '/' }; - - async.retry(100, done => { - s3.send(new ListObjectVersionsCommand(params)) - .then(data => { - if (data.Versions) { - data.Versions.forEach(version => versions.push({ - Key: version.Key, VersionId: version.VersionId })); - } - if (data.DeleteMarkers) { - data.DeleteMarkers.forEach(version => versions.push({ - Key: version.Key, VersionId: version.VersionId })); - } - if (data.IsTruncated) { - params.KeyMarker = data.NextKeyMarker; - params.VersionIdMarker = data.NextVersionIdMarker; - return done('not done yet'); - } - return done(); - }) - .catch(err => { - done(err); - }); - }, err => { - if (err) { - return done(err); - } - - assert.deepStrictEqual(versions.sort(comp), allVersions.sort(comp), - 'not same versions'); - - const objectsToDelete = versions - .filter(v => v && v.Key && v.VersionId) - .map(v => ({ - Key: String(v.Key), - VersionId: String(v.VersionId), - })); - - const deleteParams = { - Bucket: bucket, - Delete: { - Objects: objectsToDelete, - } - }; - return s3.send(new DeleteObjectsCommand(deleteParams)) - .then(() => { - done(); - }) - .catch(err => { - done(err); - }); - }); + + async.retry( + 100, + done => { + s3.send(new ListObjectVersionsCommand(params)) + .then(data => { + if (data.Versions) { + data.Versions.forEach(version => + versions.push({ + Key: version.Key, + VersionId: version.VersionId, + }), + ); + } + if (data.DeleteMarkers) { + data.DeleteMarkers.forEach(version => + versions.push({ + Key: version.Key, + VersionId: version.VersionId, + }), + ); + } + if (data.IsTruncated) { + params.KeyMarker = data.NextKeyMarker; + params.VersionIdMarker = data.NextVersionIdMarker; + return done('not done yet'); + } + return done(); + }) + .catch(err => { + done(err); + }); + }, + err => { + if (err) { + return done(err); + } + + assert.deepStrictEqual(versions.sort(comp), allVersions.sort(comp), 'not same versions'); + + const objectsToDelete = versions + .filter(v => v && v.Key && v.VersionId) + .map(v => ({ + Key: String(v.Key), + VersionId: String(v.VersionId), + })); + + const deleteParams = { + Bucket: bucket, + Delete: { + Objects: objectsToDelete, + }, + }; + return s3 + .send(new DeleteObjectsCommand(deleteParams)) + .then(() => { + done(); + }) + .catch(err => { + done(err); + }); + }, + ); }); }); - diff --git a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js index a7dc1b7ee6..d95c701ee9 100644 --- a/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js +++ b/tests/functional/aws-node-sdk/test/versioning/versioningGeneral2.js @@ -41,15 +41,14 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }) .catch(error => { assert.strictEqual(error.$metadata?.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); done(); }); }); it('should retrieve an empty versioning configuration', async () => { const params = { Bucket: bucket }; - const {$metadata, ...data} = await s3.send(new GetBucketVersioningCommand(params)); + const { $metadata, ...data } = await s3.send(new GetBucketVersioningCommand(params)); assert.strictEqual($metadata?.httpStatusCode, 200); assert.deepStrictEqual(data, {}); }); @@ -67,15 +66,14 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }) .catch(error => { assert.strictEqual(error.$metadata?.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); done(); }); }); it('should retrieve an empty versioning configuration', async () => { const params = { Bucket: bucket }; - const {$metadata, ...data} = await s3.send(new GetBucketVersioningCommand(params)); + const { $metadata, ...data } = await s3.send(new GetBucketVersioningCommand(params)); assert.strictEqual($metadata?.httpStatusCode, 200); assert.deepStrictEqual(data, {}); }); @@ -85,7 +83,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { Bucket: bucket, VersioningConfiguration: { MFADelete: 'fun', - Status: 'let\'s do it', + Status: "let's do it", }, }; s3.send(new PutBucketVersioningCommand(params)) @@ -94,15 +92,14 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { }) .catch(error => { assert.strictEqual(error.$metadata?.httpStatusCode, 400); - assert.strictEqual( - error.name, 'IllegalVersioningConfigurationException'); + assert.strictEqual(error.name, 'IllegalVersioningConfigurationException'); done(); }); }); it('should retrieve an empty versioning configuration', async () => { const params = { Bucket: bucket }; - const {$metadata, ...data} = await s3.send(new GetBucketVersioningCommand(params)); + const { $metadata, ...data } = await s3.send(new GetBucketVersioningCommand(params)); assert.strictEqual($metadata?.httpStatusCode, 200); assert.deepStrictEqual(data, {}); }); @@ -140,8 +137,7 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { return s3.send(new GetObjectCommand(params)); }) .then(data => { - assert.strictEqual(params.VersionId, data.VersionId, - 'version ids are not equal'); + assert.strictEqual(params.VersionId, data.VersionId, 'version ids are not equal'); // TODO compare the value of null version and the original // version when find out how to include value in the put params.VersionId = 'null'; @@ -156,24 +152,28 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; let nullVersionId; // create new versions - async.timesSeries(counter, (i, next) => { - s3.send(new PutObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - // get the 'null' version - return s3.send(new GetObjectCommand(paramsNull)); - }) - .then(data => { - if (nullVersionId === undefined) { - nullVersionId = data.VersionId; - } - // what to expect: nullVersionId should be the same - assert(nullVersionId, 'nullVersionId should be valid'); - assert.strictEqual(nullVersionId, data.VersionId); - next(); - }) - .catch(next); - }, done); + async.timesSeries( + counter, + (i, next) => { + s3.send(new PutObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + // get the 'null' version + return s3.send(new GetObjectCommand(paramsNull)); + }) + .then(data => { + if (nullVersionId === undefined) { + nullVersionId = data.VersionId; + } + // what to expect: nullVersionId should be the same + assert(nullVersionId, 'nullVersionId should be valid'); + assert.strictEqual(nullVersionId, data.VersionId); + next(); + }) + .catch(next); + }, + done, + ); }); it('should accept valid versioning configuration', async () => { @@ -195,32 +195,41 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { it('should update null version in versioning suspended bucket', done => { const params = { Bucket: bucket, Key: '/' }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; - - async.waterfall([ - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(() => callback()) - .catch(callback), - callback => s3.send(new PutObjectCommand(params)) - .then(() => { - versionIds.push('null'); - callback(); - }) - .catch(callback), - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - assert.strictEqual(data.VersionId, 'null', - 'version ids are equal'); - callback(); - }) - .catch(callback), - callback => s3.send(new GetObjectCommand(params)) - .then(data => { - assert.strictEqual(data.VersionId, 'null', - 'version ids are not equal'); - callback(); - }) - .catch(callback), - ], done); + + async.waterfall( + [ + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(() => callback()) + .catch(callback), + callback => + s3 + .send(new PutObjectCommand(params)) + .then(() => { + versionIds.push('null'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + assert.strictEqual(data.VersionId, 'null', 'version ids are equal'); + callback(); + }) + .catch(callback), + callback => + s3 + .send(new GetObjectCommand(params)) + .then(data => { + assert.strictEqual(data.VersionId, 'null', 'version ids are not equal'); + callback(); + }) + .catch(callback), + ], + done, + ); }); it('should enable versioning and preserve the null version', done => { @@ -233,67 +242,84 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { const params = { Bucket: bucket, Key: '/' }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; let nullVersionId; - - async.waterfall([ - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - nullVersionId = data.VersionId; - callback(); - }) - .catch(callback), - callback => s3.send(new PutBucketVersioningCommand(paramsVersioning)) - .then(() => callback()) - .catch(callback), - callback => async.timesSeries(counter, (i, next) => - s3.send(new PutObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - next(); - }) - .catch(next), - err => callback(err)), - callback => s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - assert.strictEqual(nullVersionId, data.VersionId, - 'version ids are not equal'); - callback(); - }) - .catch(callback), - ], done); + + async.waterfall( + [ + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + nullVersionId = data.VersionId; + callback(); + }) + .catch(callback), + callback => + s3 + .send(new PutBucketVersioningCommand(paramsVersioning)) + .then(() => callback()) + .catch(callback), + callback => + async.timesSeries( + counter, + (i, next) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + next(); + }) + .catch(next), + err => callback(err), + ), + callback => + s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + assert.strictEqual(nullVersionId, data.VersionId, 'version ids are not equal'); + callback(); + }) + .catch(callback), + ], + done, + ); }); it('should create delete marker and keep the null version', done => { const params = { Bucket: bucket, Key: '/' }; const paramsNull = { Bucket: bucket, Key: '/', VersionId: 'null' }; - + s3.send(new GetObjectCommand(paramsNull)) .then(data => { const nullVersionId = data.VersionId; - async.timesSeries(counter, (i, next) => { - s3.send(new DeleteObjectCommand(params)) - .then(data => { - versionIds.push(data.VersionId); - return s3.send(new GetObjectCommand(params)); - }) - .then(() => { - next(new Error('Expected NoSuchKey error')); - }) - .catch(err => { - assert.strictEqual(err.name, 'NoSuchKey'); - next(); - }); - }, err => { - if (err) { - return done(err); - } - return s3.send(new GetObjectCommand(paramsNull)) - .then(data => { - assert.strictEqual(nullVersionId, data.VersionId, - 'version ids are not equal'); - done(); - }) - .catch(done); - }); + async.timesSeries( + counter, + (i, next) => { + s3.send(new DeleteObjectCommand(params)) + .then(data => { + versionIds.push(data.VersionId); + return s3.send(new GetObjectCommand(params)); + }) + .then(() => { + next(new Error('Expected NoSuchKey error')); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchKey'); + next(); + }); + }, + err => { + if (err) { + return done(err); + } + return s3 + .send(new GetObjectCommand(paramsNull)) + .then(data => { + assert.strictEqual(nullVersionId, data.VersionId, 'version ids are not equal'); + done(); + }) + .catch(done); + }, + ); }) .catch(done); }); @@ -301,29 +327,30 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { it('should delete latest version and get the next version', done => { versionIds.reverse(); const params = { Bucket: bucket, Key: '/' }; - - async.timesSeries(versionIds.length, (i, next) => { - const versionId = versionIds[i]; - const nextVersionId = i < versionIds.length - 1 ? - versionIds[i + 1] : undefined; - const paramsVersion = - { Bucket: bucket, Key: '/', VersionId: versionId }; - - s3.send(new DeleteObjectCommand(paramsVersion)) - .then(() => s3.send(new GetObjectCommand(params))) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - if (nextVersionId !== 'null') { - assert.strictEqual(data.VersionId, nextVersionId); - } - next(); - }) - .catch(err => { - assert(err.name === 'NotFound' || - err.name === 'NoSuchKey', 'error'); - next(); - }); - }, done); + + async.timesSeries( + versionIds.length, + (i, next) => { + const versionId = versionIds[i]; + const nextVersionId = i < versionIds.length - 1 ? versionIds[i + 1] : undefined; + const paramsVersion = { Bucket: bucket, Key: '/', VersionId: versionId }; + + s3.send(new DeleteObjectCommand(paramsVersion)) + .then(() => s3.send(new GetObjectCommand(params))) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + if (nextVersionId !== 'null') { + assert.strictEqual(data.VersionId, nextVersionId); + } + next(); + }) + .catch(err => { + assert(err.name === 'NotFound' || err.name === 'NoSuchKey', 'error'); + next(); + }); + }, + done, + ); }); it('should create a bunch of objects and their versions', done => { @@ -331,38 +358,49 @@ describe('aws-node-sdk test bucket versioning', function testSuite() { const keycount = 50; const versioncount = 20; const value = '{"foo":"bar"}'; - - async.timesLimit(keycount, 10, (i, next1) => { - const key = `foo${i}`; - const params = { Bucket: bucket, Key: key, Body: value }; - async.timesLimit(versioncount, 10, (j, next2) => - s3.send(new PutObjectCommand(params)) - .then(data => { - assert(data.VersionId, 'invalid versionId'); - vids.push({ Key: key, VersionId: data.VersionId }); - next2(); - }) - .catch(next2), - next1); - }, err => { - if (err) { - return done(err); - } - assert.strictEqual(vids.length, keycount * versioncount); - const params = { - Bucket: bucket, - Delete: { - Objects: vids.map(v => ({ - Key: v.Key, - VersionId: v.VersionId, - })), - } - }; - // TODO use delete marker and check with the result - process.stdout.write('creating objects done, now deleting...'); - return s3.send(new DeleteObjectsCommand(params)) - .then(() => done()) - .catch(done); - }); + + async.timesLimit( + keycount, + 10, + (i, next1) => { + const key = `foo${i}`; + const params = { Bucket: bucket, Key: key, Body: value }; + async.timesLimit( + versioncount, + 10, + (j, next2) => + s3 + .send(new PutObjectCommand(params)) + .then(data => { + assert(data.VersionId, 'invalid versionId'); + vids.push({ Key: key, VersionId: data.VersionId }); + next2(); + }) + .catch(next2), + next1, + ); + }, + err => { + if (err) { + return done(err); + } + assert.strictEqual(vids.length, keycount * versioncount); + const params = { + Bucket: bucket, + Delete: { + Objects: vids.map(v => ({ + Key: v.Key, + VersionId: v.VersionId, + })), + }, + }; + // TODO use delete marker and check with the result + process.stdout.write('creating objects done, now deleting...'); + return s3 + .send(new DeleteObjectsCommand(params)) + .then(() => done()) + .catch(done); + }, + ); }); }); diff --git a/tests/functional/backbeat/excludedDataStoreName.js b/tests/functional/backbeat/excludedDataStoreName.js index a4b5498e8f..000f6728d5 100644 --- a/tests/functional/backbeat/excludedDataStoreName.js +++ b/tests/functional/backbeat/excludedDataStoreName.js @@ -18,12 +18,12 @@ const bucketUtil = new BucketUtility('default', {}); const s3 = bucketUtil.s3; async function getCredentials() { - const creds = await s3.config.credentials(); - const credentials = { - accessKey: creds.accessKeyId, - secretKey: creds.secretAccessKey, - }; - return credentials; + const creds = await s3.config.credentials(); + const credentials = { + accessKey: creds.accessKeyId, + secretKey: creds.secretAccessKey, + }; + return credentials; } async function getS3Hostname() { @@ -38,87 +38,96 @@ describe('excludedDataStoreName', () => { let location1; let location2; - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - next(); - }) - .catch(next); - }, - next => { - getS3Hostname() - .then(hostname => { - s3Hostname = hostname; - location1 = config.restEndpoints[s3Hostname] || config.restEndpoints.localhost; - location2 = 'us-east-2'; - next(); - }) - .catch(next); - }, - next => { - s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(data => { - expectedVersions.push(data.VersionId); - next(); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(data => { - const versionId = data.VersionId; - return updateMetadata( - { bucket: testBucket, objectKey: 'key0', versionId, authCredentials: credentials }, - { dataStoreName: location2 }, - next); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(data => { - expectedVersions.push(data.VersionId); - next(); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1' })) - .then(data => { - const versionId = data.VersionId; - return updateMetadata( - { bucket: testBucket, objectKey: 'key1', versionId, authCredentials: credentials }, - { dataStoreName: location2 }, - next); - }) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2' })) - .then(() => next()) - .catch(next); - }, - ], done)); + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next); + }, + next => { + getS3Hostname() + .then(hostname => { + s3Hostname = hostname; + location1 = config.restEndpoints[s3Hostname] || config.restEndpoints.localhost; + location2 = 'us-east-2'; + next(); + }) + .catch(next); + }, + next => { + s3.send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(data => { + expectedVersions.push(data.VersionId); + next(); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(data => { + const versionId = data.VersionId; + return updateMetadata( + { bucket: testBucket, objectKey: 'key0', versionId, authCredentials: credentials }, + { dataStoreName: location2 }, + next, + ); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(data => { + expectedVersions.push(data.VersionId); + next(); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key0' })) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1' })) + .then(data => { + const versionId = data.VersionId; + return updateMetadata( + { bucket: testBucket, objectKey: 'key1', versionId, authCredentials: credentials }, + { dataStoreName: location2 }, + next, + ); + }) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2' })) + .then(() => next()) + .catch(next); + }, + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -126,183 +135,182 @@ describe('excludedDataStoreName', () => { }); it('should return error when listing current versions if excluded-data-store-name is not in config', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'excluded-data-store-name': 'idonotexist' }, - authCredentials: credentials, - }, err => { - assert(err, 'Expected error but found none'); - assert.strictEqual(err.code, 'InvalidLocationConstraint'); - assert.strictEqual(err.statusCode, 400); - assert.strictEqual(err.message, 'value of the location you are attempting to set' + - ' - idonotexist - is not listed in the locationConstraint config'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'excluded-data-store-name': 'idonotexist' }, + authCredentials: credentials, + }, + err => { + assert(err, 'Expected error but found none'); + assert.strictEqual(err.code, 'InvalidLocationConstraint'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual( + err.message, + 'value of the location you are attempting to set' + + ' - idonotexist - is not listed in the locationConstraint config', + ); + return done(); + }, + ); }); it('should return error when listing non-current versions if excluded-data-store-name is not in config', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': 'idonotexist' }, - authCredentials: credentials, - }, err => { - assert(err, 'Expected error but found none'); - assert.strictEqual(err.code, 'InvalidLocationConstraint'); - assert.strictEqual(err.statusCode, 400); - assert.strictEqual(err.message, 'value of the location you are attempting to set' + - ' - idonotexist - is not listed in the locationConstraint config'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': 'idonotexist' }, + authCredentials: credentials, + }, + err => { + assert(err, 'Expected error but found none'); + assert.strictEqual(err.code, 'InvalidLocationConstraint'); + assert.strictEqual(err.statusCode, 400); + assert.strictEqual( + err.message, + 'value of the location you are attempting to set' + + ' - idonotexist - is not listed in the locationConstraint config', + ); + return done(); + }, + ); }); it('should exclude current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2 }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[1].Key, 'key2'); - assert.strictEqual(contents[1].DataStoreName, location1); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2 }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[0].DataStoreName, location1); + assert.strictEqual(contents[1].Key, 'key2'); + assert.strictEqual(contents[1].DataStoreName, location1); + return done(); + }, + ); }); it('should return trucated listing that excludes current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2, 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key0'); - assert.strictEqual(data.MaxKeys, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - - return makeBackbeatRequest({ + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'excluded-data-store-name': location2, - 'max-keys': '1', - 'marker': 'key0', - }, + queryObj: { 'list-type': 'current', 'excluded-data-store-name': location2, 'max-keys': '1' }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'key0'); assert.strictEqual(data.MaxKeys, 1); const contents = data.Contents; assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key2'); + assert.strictEqual(contents[0].Key, 'key0'); assert.strictEqual(contents[0].DataStoreName, location1); - return done(); - }); - }); + + return makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'current', + 'excluded-data-store-name': location2, + 'max-keys': '1', + marker: 'key0', + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + + assert.strictEqual(contents[0].Key, 'key2'); + assert.strictEqual(contents[0].DataStoreName, location1); + return done(); + }, + ); + }, + ); }); it('should exclude non-current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2 }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert(!data.NextVersionIdMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[0].VersionId, expectedVersions[1]); - assert.strictEqual(contents[1].Key, 'key0'); - assert.strictEqual(contents[1].DataStoreName, location1); - assert.strictEqual(contents[1].VersionId, expectedVersions[0]); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2 }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert(!data.NextVersionIdMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[0].DataStoreName, location1); + assert.strictEqual(contents[0].VersionId, expectedVersions[1]); + assert.strictEqual(contents[1].Key, 'key0'); + assert.strictEqual(contents[1].DataStoreName, location1); + assert.strictEqual(contents[1].VersionId, expectedVersions[0]); + return done(); + }, + ); }); it('should return trucated listing that excludes non-current versions stored in location2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2, 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextKeyMarker, 'key0'); - assert.strictEqual(data.NextVersionIdMarker, expectedVersions[1]); - assert.strictEqual(data.MaxKeys, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[0].VersionId, expectedVersions[1]); - return makeBackbeatRequest({ + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { - 'list-type': 'noncurrent', - 'excluded-data-store-name': location2, - 'key-marker': 'key0', - 'version-id-marker': expectedVersions[1], - 'max-keys': '1', - }, + queryObj: { 'list-type': 'noncurrent', 'excluded-data-store-name': location2, 'max-keys': '1' }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); assert.strictEqual(data.IsTruncated, true); assert.strictEqual(data.NextKeyMarker, 'key0'); - assert.strictEqual(data.NextVersionIdMarker, expectedVersions[0]); + assert.strictEqual(data.NextVersionIdMarker, expectedVersions[1]); assert.strictEqual(data.MaxKeys, 1); const contents = data.Contents; @@ -310,33 +318,67 @@ describe('excludedDataStoreName', () => { assert.strictEqual(contents[0].Key, 'key0'); assert.strictEqual(contents[0].DataStoreName, location1); - assert.strictEqual(contents[0].VersionId, expectedVersions[0]); - return makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'noncurrent', - 'excluded-data-store-name': location2, - 'key-marker': 'key0', - 'version-id-marker': expectedVersions[0], - 'max-keys': '1', + assert.strictEqual(contents[0].VersionId, expectedVersions[1]); + return makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'noncurrent', + 'excluded-data-store-name': location2, + 'key-marker': 'key0', + 'version-id-marker': expectedVersions[1], + 'max-keys': '1', + }, + authCredentials: credentials, }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert(!data.NextVersionIdMarker); - assert.strictEqual(data.MaxKeys, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 0); - return done(); - }); - }); - }); + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextKeyMarker, 'key0'); + assert.strictEqual(data.NextVersionIdMarker, expectedVersions[0]); + assert.strictEqual(data.MaxKeys, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[0].DataStoreName, location1); + assert.strictEqual(contents[0].VersionId, expectedVersions[0]); + return makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'noncurrent', + 'excluded-data-store-name': location2, + 'key-marker': 'key0', + 'version-id-marker': expectedVersions[0], + 'max-keys': '1', + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert(!data.NextVersionIdMarker); + assert.strictEqual(data.MaxKeys, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 0); + return done(); + }, + ); + }, + ); + }, + ); }); }); diff --git a/tests/functional/backbeat/listDeleteMarker.js b/tests/functional/backbeat/listDeleteMarker.js index d0ee845bdd..b54072b641 100644 --- a/tests/functional/backbeat/listDeleteMarker.js +++ b/tests/functional/backbeat/listDeleteMarker.js @@ -26,41 +26,57 @@ async function getCredentials() { return credentials; } - describe('listLifecycle with non-current delete marker', () => { let expectedVersionId; let expectedDMVersionId; const testBucket = 'bucket-for-list-lifecycle-noncurrent-dm-tests'; const keyName = 'key0'; - before(done => async.series([ - next => getCredentials().then(creds => { - credentials = creds; - next(); - }).catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(data => { - expectedDMVersionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(data => { - expectedVersionId = data.VersionId; - next(); - }) - .catch(next), - ], done)); - + before(done => + async.series( + [ + next => + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(data => { + expectedDMVersionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(data => { + expectedVersionId = data.VersionId; + next(); + }) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -68,66 +84,75 @@ describe('listLifecycle with non-current delete marker', () => { }); it('should return the current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedVersionId); + return done(); + }, + ); }); it('should return the non-current delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedDMVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedDMVersionId); + return done(); + }, + ); }); it('should return no orphan delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); }); @@ -136,26 +161,41 @@ describe('listLifecycle with current delete marker version', () => { const testBucket = 'bucket-for-list-lifecycle-current-dm-tests'; const keyName = 'key0'; - before(done => async.series([ - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(data => { - expectedVersionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) - .then(() => next()) - .catch(next), - ], done)); + before(done => + async.series( + [ + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(data => { + expectedVersionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName })) + .then(() => next()) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -163,62 +203,71 @@ describe('listLifecycle with current delete marker version', () => { }); it('should return no current object if current version is a delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); it('should return the non-current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedVersionId); + return done(); + }, + ); }); it('should return no orphan delete marker', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/listLifecycleCurrents.js b/tests/functional/backbeat/listLifecycleCurrents.js index 5b83270a82..7e4821bf51 100644 --- a/tests/functional/backbeat/listLifecycleCurrents.js +++ b/tests/functional/backbeat/listLifecycleCurrents.js @@ -52,10 +52,12 @@ function checkContents(contents, expectedKeyVersions) { assert(d.Owner.ID); assert(d.StorageClass); assert.strictEqual(d.StorageClass, 'STANDARD'); - assert.deepStrictEqual(d.TagSet, [{ - Key: 'mykey', - Value: 'myvalue', - }]); + assert.deepStrictEqual(d.TagSet, [ + { + Key: 'mykey', + Value: 'myvalue', + }, + ]); assert.strictEqual(d.IsLatest, true); assert.strictEqual(d.DataStoreName, location); assert.strictEqual(d.ListType, 'current'); @@ -71,540 +73,685 @@ function checkContents(contents, expectedKeyVersions) { let date; const expectedKeyVersions = {}; - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - return getS3Hostname(); - }) - .then(() => next()) - .catch(next); - }, - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: emptyBucket })) - .then(() => next()) - .catch(next), - next => { - if (versioning !== 'Enabled') { - return process.nextTick(next); - } - return s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next); - }, - next => { - if (versioning !== 'Enabled') { - return process.nextTick(next); - } - return s3.send(new PutBucketVersioningCommand({ - Bucket: emptyBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next); - }, - next => async.times(3, (n, cb) => { - const keyName = `oldkey${n}`; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(cb); - }, next), - next => { - date = new Date(Date.now()).toISOString(); - return async.times(5, (n, cb) => { - const keyName = `key${n}`; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + return getS3Hostname(); }) - .catch(cb); - }, next); - }, - ], done)); + .then(() => next()) + .catch(next); + }, + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: emptyBucket })) + .then(() => next()) + .catch(next), + next => { + if (versioning !== 'Enabled') { + return process.nextTick(next); + } + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => { + if (versioning !== 'Enabled') { + return process.nextTick(next); + } + return s3 + .send( + new PutBucketVersioningCommand({ + Bucket: emptyBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next); + }, + next => + async.times( + 3, + (n, cb) => { + const keyName = `oldkey${n}`; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(cb); + }, + next, + ), + next => { + date = new Date(Date.now()).toISOString(); + return async.times( + 5, + (n, cb) => { + const keyName = `key${n}`; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(cb); + }, + next, + ); + }, + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); await s3.send(new DeleteBucketCommand({ Bucket: testBucket })); - await s3.send(new DeleteBucketCommand({ Bucket: emptyBucket })); + await s3.send(new DeleteBucketCommand({ Bucket: emptyBucket })); }); it('should return empty list of current versions if bucket is empty', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: emptyBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: emptyBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); it('should return empty list of current versions if prefix does not apply', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'prefix': 'unknown' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', prefix: 'unknown' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 0); + return done(); + }, + ); }); it('should return empty list if max-keys is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': '0' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 0); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-keys': '0' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 0); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return NoSuchBucket error if bucket does not exist', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: 'idonotexist', - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'NoSuchBucket'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: 'idonotexist', + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'NoSuchBucket'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-keys is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-keys': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '0' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '0' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument if max-scanned-lifecycle-listing-entries exceeds the default value', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': - (config.maxScannedLifecycleListingEntries + 1).toString() }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'current', + 'max-scanned-lifecycle-listing-entries': ( + config.maxScannedLifecycleListingEntries + 1 + ).toString(), + }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return all the current versions', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - - const contents = data.Contents; - assert.strictEqual(contents.length, 8); - checkContents(contents, expectedKeyVersions); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + + const contents = data.Contents; + assert.strictEqual(contents.length, 8); + checkContents(contents, expectedKeyVersions); + + return done(); + }, + ); }); it('should return all the current versions before max scanned entries value is reached', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '5' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-scanned-lifecycle-listing-entries': '5' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key4'); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, 5); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'key4'); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, 5); - const contents = data.Contents; - assert.strictEqual(contents.length, 5); - checkContents(contents, expectedKeyVersions); + const contents = data.Contents; + assert.strictEqual(contents.length, 5); + checkContents(contents, expectedKeyVersions); - return done(); - }); + return done(); + }, + ); }); it('should return all the current versions with prefix old', done => { const prefix = 'old'; - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', prefix }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', prefix }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Prefix, prefix); + + const contents = data.Contents; + assert.strictEqual(contents.length, 3); + checkContents(contents, expectedKeyVersions); + + return done(); + }, + ); + }); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Prefix, prefix); + it('should return the current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 3); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey0'); + assert.strictEqual(contents[1].Key, 'oldkey1'); + assert.strictEqual(contents[2].Key, 'oldkey2'); + return done(); + }, + ); + }); - const contents = data.Contents; - assert.strictEqual(contents.length, 3); - checkContents(contents, expectedKeyVersions); + it('should truncate list of current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'oldkey0'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey0'); + return done(); + }, + ); + }); - return done(); - }); + it('should return the next truncate list of current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', marker: 'oldkey0' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.Marker, 'oldkey0'); + assert.strictEqual(data.NextMarker, 'oldkey1'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey1'); + assert.strictEqual(data.BeforeDate, date); + return done(); + }, + ); }); - it('should return the current versions before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); + it('should return the last truncate list of current versions before a defined date', done => { + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', marker: 'oldkey1' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual( + data.MaxScannedLifecycleListingEntries, + config.maxScannedLifecycleListingEntries, + ); + assert.strictEqual(data.Marker, 'oldkey1'); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, 'oldkey2'); + return done(); + }, + ); + }); + }); +}); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 3); - assert.strictEqual(data.BeforeDate, date); +describe('listLifecycleCurrents with bucket versioning enabled and maxKeys', () => { + const testBucket = 'bucket-for-list-lifecycle-current-tests-truncated'; + const expectedKeyVersions = {}; - const contents = data.Contents; - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey0'); - assert.strictEqual(contents[1].Key, 'oldkey1'); - assert.strictEqual(contents[2].Key, 'oldkey2'); - return done(); - }); - }); + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + return getS3Hostname(); + }) + .then(() => next()) + .catch(next); + }, + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + async.times( + 3, + (n, cb) => { + const keyName = 'key0'; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(err => cb(err)); + }, + next, + ), + next => + async.times( + 5, + (n, cb) => { + const keyName = 'key1'; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(err => cb(err)); + }, + next, + ), + next => + async.times( + 3, + (n, cb) => { + const keyName = 'key2'; + s3.send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName] = data.VersionId; + cb(); + }) + .catch(err => cb(err)); + }, + next, + ), + ], + done, + ), + ); - it('should truncate list of current versions before a defined date', done => { - makeBackbeatRequest({ + after(async () => { + await removeAllVersionsPromise({ Bucket: testBucket }); + await s3.send(new DeleteBucketCommand({ Bucket: testBucket })); + }); + + it('should return truncated lists - part 1', done => { + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1' }, + queryObj: { 'list-type': 'current', 'max-keys': '1' }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'oldkey0'); + assert.strictEqual(data.NextMarker, 'key0'); assert.strictEqual(data.MaxKeys, 1); assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.BeforeDate, date); assert.strictEqual(data.Contents.length, 1); const contents = data.Contents; + assert.strictEqual(contents.length, 1); checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey0'); + assert.strictEqual(contents[0].Key, 'key0'); return done(); - }); - }); + }, + ); + }); - it('should return the next truncate list of current versions before a defined date', done => { - makeBackbeatRequest({ + it('should return truncated lists - part 2', done => { + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', 'marker': 'oldkey0' }, + queryObj: { + 'list-type': 'current', + 'max-keys': '1', + marker: 'key0', + }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); + assert.strictEqual(data.Marker, 'key0'); assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.Marker, 'oldkey0'); - assert.strictEqual(data.NextMarker, 'oldkey1'); + assert.strictEqual(data.NextMarker, 'key1'); assert.strictEqual(data.MaxKeys, 1); assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); assert.strictEqual(data.Contents.length, 1); const contents = data.Contents; + assert.strictEqual(contents.length, 1); checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey1'); - assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(contents[0].Key, 'key1'); return done(); - }); - }); + }, + ); + }); - it('should return the last truncate list of current versions before a defined date', done => { - makeBackbeatRequest({ + it('should return truncated lists - part 3', done => { + makeBackbeatRequest( + { method: 'GET', bucket: testBucket, - queryObj: { 'list-type': 'current', 'before-date': date, 'max-keys': '1', 'marker': 'oldkey1' }, + queryObj: { + 'list-type': 'current', + 'max-keys': '1', + marker: 'key1', + }, authCredentials: credentials, - }, (err, response) => { + }, + (err, response) => { assert.ifError(err); assert.strictEqual(response.statusCode, 200); const data = JSON.parse(response.body); + assert(!data.NextMarker); assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.Marker, 'key1'); assert.strictEqual(data.MaxKeys, 1); assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Marker, 'oldkey1'); - assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.Contents.length, 1); const contents = data.Contents; assert.strictEqual(contents.length, 1); checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'oldkey2'); + assert.strictEqual(contents[0].Key, 'key2'); return done(); - }); - }); - }); -}); - -describe('listLifecycleCurrents with bucket versioning enabled and maxKeys', () => { - const testBucket = 'bucket-for-list-lifecycle-current-tests-truncated'; - const expectedKeyVersions = {}; - - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - return getS3Hostname(); - }) - .then(() => next()) - .catch(next); - }, - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => async.times(3, (n, cb) => { - const keyName = 'key0'; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(err => cb(err)); - }, next), - next => async.times(5, (n, cb) => { - const keyName = 'key1'; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(err => cb(err)); - }, next), - next => async.times(3, (n, cb) => { - const keyName = 'key2'; - s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName] = data.VersionId; - cb(); - }) - .catch(err => cb(err)); - }, next), - ], done)); - - after(async () => { - await removeAllVersionsPromise({ Bucket: testBucket }); - await s3.send(new DeleteBucketCommand({ Bucket: testBucket })); - }); - - it('should return truncated lists - part 1', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key0'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'key0'); - return done(); - }); - }); - - it('should return truncated lists - part 2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'max-keys': '1', - 'marker': 'key0', - }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.Marker, 'key0'); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key1'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'key1'); - return done(); - }); - }); - - it('should return truncated lists - part 3', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'max-keys': '1', - 'marker': 'key1', }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert(!data.NextMarker); - assert.strictEqual(data.IsTruncated, false); - assert.strictEqual(data.Marker, 'key1'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, 'key2'); - return done(); - }); + ); }); }); @@ -615,74 +762,107 @@ describe('listLifecycleCurrents with bucket versioning enabled and delete object const keyName2 = 'key2'; const expectedKeyVersions = {}; - before(done => async.series([ - next => { - getCredentials() - .then(creds => { - credentials = creds; - return getS3Hostname(); - }) - .then(() => next()) - .catch(next); - }, - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName0, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName0] = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName1, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName1 })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName2, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => { - expectedKeyVersions[keyName2] = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutObjectCommand({ - Bucket: testBucket, - Key: keyName2, - Body: '123', - Tagging: 'mykey=myvalue', - })) - .then(data => s3.send(new DeleteObjectCommand({ - Bucket: testBucket, - Key: keyName2, - VersionId: data.VersionId, - })) - .then(() => next()) - .catch(next)) - .catch(next), - ], done)); - + before(done => + async.series( + [ + next => { + getCredentials() + .then(creds => { + credentials = creds; + return getS3Hostname(); + }) + .then(() => next()) + .catch(next); + }, + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName0, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName0] = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName1, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: testBucket, Key: keyName1 })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName2, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + expectedKeyVersions[keyName2] = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send( + new PutObjectCommand({ + Bucket: testBucket, + Key: keyName2, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => + s3 + .send( + new DeleteObjectCommand({ + Bucket: testBucket, + Key: keyName2, + VersionId: data.VersionId, + }), + ) + .then(() => next()) + .catch(next), + ) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -690,57 +870,63 @@ describe('listLifecycleCurrents with bucket versioning enabled and delete object }); it('should return truncated lists - part 1', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current', 'max-keys': '1' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, keyName0); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, keyName0); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current', 'max-keys': '1' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, keyName0); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, keyName0); + return done(); + }, + ); }); it('should return truncated lists - part 2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'current', - 'max-keys': '1', - 'marker': keyName0, + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'current', + 'max-keys': '1', + marker: keyName0, + }, + authCredentials: credentials, }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert(!data.NextMarker); - assert.strictEqual(data.IsTruncated, false); - assert.strictEqual(data.Marker, keyName0); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents, expectedKeyVersions); - assert.strictEqual(contents[0].Key, keyName2); - return done(); - }); + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert(!data.NextMarker); + assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.Marker, keyName0); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents, expectedKeyVersions); + assert.strictEqual(contents[0].Key, keyName2); + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js b/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js index 7eb9608642..5e08d2d642 100644 --- a/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js +++ b/tests/functional/backbeat/listLifecycleOrphanDeleteMarkers.js @@ -49,84 +49,135 @@ function checkContents(contents) { } function createDeleteMarker(s3, bucketName, keyName, cb) { - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue' - })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) - .then(() => next()) - .catch(next), - ], cb); + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) + .then(() => next()) + .catch(next), + ], + cb, + ); } function createOrphanDeleteMarker(s3, bucketName, keyName, cb) { let versionId; - return async.series([ - next => s3.send(new PutObjectCommand({ - Bucket: bucketName, - Key: keyName, - Body: '123', - Tagging: 'mykey=myvalue' - })) - .then(data => { - versionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) - .then(() => next()) - .catch(next), - next => s3.send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName, VersionId: versionId })) - .then(() => next()) - .catch(next), - ], cb); + return async.series( + [ + next => + s3 + .send( + new PutObjectCommand({ + Bucket: bucketName, + Key: keyName, + Body: '123', + Tagging: 'mykey=myvalue', + }), + ) + .then(data => { + versionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new DeleteObjectCommand({ Bucket: bucketName, Key: keyName, VersionId: versionId })) + .then(() => next()) + .catch(next), + ], + cb, + ); } describe('listLifecycleOrphanDeleteMarkers', () => { let date; - before(done => async.series([ - next => getCredentials().then(creds => { - credentials = creds; - next(); - }).catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: emptyBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: nonVersionedBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: emptyBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => async.times(3, (n, cb) => { - createOrphanDeleteMarker(s3, testBucket, `key${n}old`, cb); - }, next), - next => createDeleteMarker(s3, testBucket, 'no-orphan-delete-marker', next), - next => { - date = new Date(Date.now()).toISOString(); - return async.times(5, (n, cb) => { - createOrphanDeleteMarker(s3, testBucket, `key${n}`, cb); - }, next); - }, - ], done)); + before(done => + async.series( + [ + next => + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: emptyBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: nonVersionedBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: emptyBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + async.times( + 3, + (n, cb) => { + createOrphanDeleteMarker(s3, testBucket, `key${n}old`, cb); + }, + next, + ), + next => createDeleteMarker(s3, testBucket, 'no-orphan-delete-marker', next), + next => { + date = new Date(Date.now()).toISOString(); + return async.times( + 5, + (n, cb) => { + createOrphanDeleteMarker(s3, testBucket, `key${n}`, cb); + }, + next, + ); + }, + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -136,409 +187,465 @@ describe('listLifecycleOrphanDeleteMarkers', () => { }); it('should return empty list of orphan delete markers if bucket is empty', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: emptyBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: emptyBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return empty list of orphan delete markers if prefix does not apply', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'prefix': 'unknown' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', prefix: 'unknown' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return empty list if max-keys is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-keys': '0' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 0); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-keys': '0' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 0); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 0); + + return done(); + }, + ); }); it('should return InvalidArgument error if max-keys is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-keys': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-keys': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is invalid', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': 'a' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': 'a' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is set to 0', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '0' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '0' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument error if max-scanned-lifecycle-listing-entries is set to 2', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '2' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '2' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return InvalidArgument if max-scanned-lifecycle-listing-entries exceeds the default value', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': - (config.maxScannedLifecycleListingEntries + 1).toString() }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'orphan', + 'max-scanned-lifecycle-listing-entries': (config.maxScannedLifecycleListingEntries + 1).toString(), + }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + return done(); + }, + ); }); it('should return error if bucket does not exist', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: 'idonotexist', - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, err => { - assert.strictEqual(err.code, 'NoSuchBucket'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: 'idonotexist', + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + err => { + assert.strictEqual(err.code, 'NoSuchBucket'); + return done(); + }, + ); }); it('should return all the orphan delete markers', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - - const contents = data.Contents; - assert.strictEqual(contents.length, 8); - checkContents(contents); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + + const contents = data.Contents; + assert.strictEqual(contents.length, 8); + checkContents(contents); + + return done(); + }, + ); }); it('should only return delete marker that passed the full keys evaluation to prevent false positives', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '4' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, 4); - - // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, - // which results in a difference in the number of keys scanned and, consequently, - // affects the value of the NextMarker and Contents. - const contents = data.Contents; - const nextMarker = data.NextMarker; - - if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { - // With v1 metadata bucket key format, master key is automaticaly deleted - // when the last version of an object is a delete marker - assert.strictEqual(nextMarker, 'key1'); - assert.strictEqual(contents.length, 3); - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[1].Key, 'key0old'); - assert.strictEqual(contents[2].Key, 'key1'); - } else { - assert.strictEqual(nextMarker, 'key0'); - assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key0'); - } - checkContents(contents); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '4' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, 4); + + // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, + // which results in a difference in the number of keys scanned and, consequently, + // affects the value of the NextMarker and Contents. + const contents = data.Contents; + const nextMarker = data.NextMarker; + + if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { + // With v1 metadata bucket key format, master key is automaticaly deleted + // when the last version of an object is a delete marker + assert.strictEqual(nextMarker, 'key1'); + assert.strictEqual(contents.length, 3); + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[1].Key, 'key0old'); + assert.strictEqual(contents[2].Key, 'key1'); + } else { + assert.strictEqual(nextMarker, 'key0'); + assert.strictEqual(contents.length, 1); + assert.strictEqual(contents[0].Key, 'key0'); + } + checkContents(contents); + + return done(); + }, + ); }); it('should return all the orphan delete markers before max scanned entries value is reached', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '3' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, 3); - - // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, - // which results in a difference in the number of keys scanned and, consequently, - // affects the value of the NextMarker and Contents. - const contents = data.Contents; - const nextMarker = data.NextMarker; - - if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { - // With v1 metadata bucket key format, master key is automaticaly deleted - // when the last version of an object is a delete marker - assert.strictEqual(nextMarker, 'key0old'); - assert.strictEqual(contents.length, 2); - assert.strictEqual(contents[0].Key, 'key0'); - assert.strictEqual(contents[1].Key, 'key0old'); - } else { - assert.strictEqual(nextMarker, 'key0'); - assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key0'); - } - checkContents(contents); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'max-scanned-lifecycle-listing-entries': '3' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, 3); + + // Depending on the metadata bucket key format, the orphan delete marker is denoted by 1 or 2 entries, + // which results in a difference in the number of keys scanned and, consequently, + // affects the value of the NextMarker and Contents. + const contents = data.Contents; + const nextMarker = data.NextMarker; + + if (process.env.DEFAULT_BUCKET_KEY_FORMAT === 'v1') { + // With v1 metadata bucket key format, master key is automaticaly deleted + // when the last version of an object is a delete marker + assert.strictEqual(nextMarker, 'key0old'); + assert.strictEqual(contents.length, 2); + assert.strictEqual(contents[0].Key, 'key0'); + assert.strictEqual(contents[1].Key, 'key0old'); + } else { + assert.strictEqual(nextMarker, 'key0'); + assert.strictEqual(contents.length, 1); + assert.strictEqual(contents[0].Key, 'key0'); + } + checkContents(contents); + + return done(); + }, + ); }); it('should return all the orphan delete markers with prefix key1', done => { const prefix = 'key1'; - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', prefix }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Prefix, prefix); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key1'); - assert.strictEqual(contents[1].Key, 'key1old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', prefix }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Prefix, prefix); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key1'); + assert.strictEqual(contents[1].Key, 'key1old'); + + return done(); + }, + ); }); it('should return the orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'orphan', - 'before-date': date, - }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 3); - assert.strictEqual(data.BeforeDate, date); - - const contents = data.Contents; - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key0old'); - assert.strictEqual(contents[1].Key, 'key1old'); - assert.strictEqual(contents[2].Key, 'key2old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'orphan', + 'before-date': date, + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 3); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key0old'); + assert.strictEqual(contents[1].Key, 'key1old'); + assert.strictEqual(contents[2].Key, 'key2old'); + + return done(); + }, + ); }); it('should truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { - 'list-type': 'orphan', - 'before-date': date, - 'max-keys': '1', - }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.NextMarker, 'key0old'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.BeforeDate, date); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key0old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { + 'list-type': 'orphan', + 'before-date': date, + 'max-keys': '1', + }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.NextMarker, 'key0old'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key0old'); + + return done(); + }, + ); }); it('should return the second truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', 'marker': 'key0old' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.Marker, 'key0old'); - assert.strictEqual(data.NextMarker, 'key1old'); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Contents.length, 1); - - const contents = data.Contents; - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key1old'); - assert.strictEqual(data.BeforeDate, date); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', marker: 'key0old' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.Marker, 'key0old'); + assert.strictEqual(data.NextMarker, 'key1old'); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Contents.length, 1); + + const contents = data.Contents; + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key1old'); + assert.strictEqual(data.BeforeDate, date); + + return done(); + }, + ); }); it('should return the third truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', 'marker': 'key1old' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, true); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Marker, 'key1old'); - assert.strictEqual(data.BeforeDate, date); - assert.strictEqual(data.NextMarker, 'key2old'); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - checkContents(contents); - assert.strictEqual(contents[0].Key, 'key2old'); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', marker: 'key1old' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, true); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Marker, 'key1old'); + assert.strictEqual(data.BeforeDate, date); + assert.strictEqual(data.NextMarker, 'key2old'); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + checkContents(contents); + assert.strictEqual(contents[0].Key, 'key2old'); + + return done(); + }, + ); }); it('should return the fourth and last truncate list of orphan delete markers before a defined date', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', 'marker': 'key2old' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - - const data = JSON.parse(response.body); - assert.strictEqual(data.IsTruncated, false); - assert.strictEqual(data.MaxKeys, 1); - assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); - assert.strictEqual(data.Marker, 'key2old'); - assert.strictEqual(data.BeforeDate, date); - - const contents = data.Contents; - assert.strictEqual(contents.length, 0); - - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'orphan', 'before-date': date, 'max-keys': '1', marker: 'key2old' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + + const data = JSON.parse(response.body); + assert.strictEqual(data.IsTruncated, false); + assert.strictEqual(data.MaxKeys, 1); + assert.strictEqual(data.MaxScannedLifecycleListingEntries, config.maxScannedLifecycleListingEntries); + assert.strictEqual(data.Marker, 'key2old'); + assert.strictEqual(data.BeforeDate, date); + + const contents = data.Contents; + assert.strictEqual(contents.length, 0); + + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/listNullVersion.js b/tests/functional/backbeat/listNullVersion.js index 09f092f7a8..05f2a8841d 100644 --- a/tests/functional/backbeat/listNullVersion.js +++ b/tests/functional/backbeat/listNullVersion.js @@ -31,44 +31,68 @@ async function getCredentials() { describe('listLifecycle if null version', () => { let versionForKey2; - before(done => async.series([ - next => getCredentials().then(creds => { - credentials = creds; - next(); - }).catch(next), - next => s3.send(new CreateBucketCommand({ Bucket: testBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: testBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) - .then(data => - // delete version to create a null current version for key1. - s3.send(new DeleteObjectCommand({ - Bucket: testBucket, - Key: 'key1', - VersionId: data.VersionId - })) - ) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) - .then(data => { - versionForKey2 = data.VersionId; - next(); - }) - .catch(next), - ], done)); + before(done => + async.series( + [ + next => + getCredentials() + .then(creds => { + credentials = creds; + next(); + }) + .catch(next), + next => + s3 + .send(new CreateBucketCommand({ Bucket: testBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: testBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key1', Body: '123' })) + .then(data => + // delete version to create a null current version for key1. + s3.send( + new DeleteObjectCommand({ + Bucket: testBucket, + Key: 'key1', + VersionId: data.VersionId, + }), + ), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: testBucket, Key: 'key2', Body: '123' })) + .then(data => { + versionForKey2 = data.VersionId; + next(); + }) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: testBucket }); @@ -76,55 +100,61 @@ describe('listLifecycle if null version', () => { }); it('should return the null noncurrent versions', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 1); - assert.strictEqual(contents[0].Key, 'key2'); - assert.strictEqual(contents[0].VersionId, 'null'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 1); + assert.strictEqual(contents[0].Key, 'key2'); + assert.strictEqual(contents[0].VersionId, 'null'); + return done(); + }, + ); }); it('should return the null current versions', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: testBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - - const contents = data.Contents; - assert.strictEqual(contents.length, 2); - - const firstKey = contents[0]; - assert.strictEqual(firstKey.Key, 'key1'); - assert.strictEqual(firstKey.VersionId, 'null'); - - const secondKey = contents[1]; - assert.strictEqual(secondKey.Key, 'key2'); - assert.strictEqual(secondKey.VersionId, versionForKey2); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: testBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + + const contents = data.Contents; + assert.strictEqual(contents.length, 2); + + const firstKey = contents[0]; + assert.strictEqual(firstKey.Key, 'key1'); + assert.strictEqual(firstKey.VersionId, 'null'); + + const secondKey = contents[1]; + assert.strictEqual(secondKey.Key, 'key2'); + assert.strictEqual(secondKey.VersionId, versionForKey2); + return done(); + }, + ); }); }); @@ -133,32 +163,51 @@ describe('listLifecycle with null current version after versioning suspended', ( const nullObjectBucket = 'bucket-for-list-lifecycle-current-null-tests'; const keyName = 'key0'; - before(done => async.series([ - next => s3.send(new CreateBucketCommand({ Bucket: nullObjectBucket })) - .then(() => next()) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: nullObjectBucket, - VersioningConfiguration: { Status: 'Enabled' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) - .then(data => { - expectedVersionId = data.VersionId; - next(); - }) - .catch(next), - next => s3.send(new PutBucketVersioningCommand({ - Bucket: nullObjectBucket, - VersioningConfiguration: { Status: 'Suspended' }, - })) - .then(() => next()) - .catch(next), - next => s3.send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) - .then(() => next()) - .catch(next), - ], done)); + before(done => + async.series( + [ + next => + s3 + .send(new CreateBucketCommand({ Bucket: nullObjectBucket })) + .then(() => next()) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: nullObjectBucket, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) + .then(data => { + expectedVersionId = data.VersionId; + next(); + }) + .catch(next), + next => + s3 + .send( + new PutBucketVersioningCommand({ + Bucket: nullObjectBucket, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ) + .then(() => next()) + .catch(next), + next => + s3 + .send(new PutObjectCommand({ Bucket: nullObjectBucket, Key: keyName })) + .then(() => next()) + .catch(next), + ], + done, + ), + ); after(async () => { await removeAllVersionsPromise({ Bucket: nullObjectBucket }); @@ -166,46 +215,52 @@ describe('listLifecycle with null current version after versioning suspended', ( }); it('should return list of current versions when bucket has a null current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: nullObjectBucket, - queryObj: { 'list-type': 'current' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, 'null'); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: nullObjectBucket, + queryObj: { 'list-type': 'current' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, 'null'); + return done(); + }, + ); }); it('should return list of non-current versions when bucket has a null current version', done => { - makeBackbeatRequest({ - method: 'GET', - bucket: nullObjectBucket, - queryObj: { 'list-type': 'noncurrent' }, - authCredentials: credentials, - }, (err, response) => { - assert.ifError(err); - assert.strictEqual(response.statusCode, 200); - const data = JSON.parse(response.body); - - assert.strictEqual(data.IsTruncated, false); - assert(!data.NextKeyMarker); - assert.strictEqual(data.MaxKeys, 1000); - assert.strictEqual(data.Contents.length, 1); - const key = data.Contents[0]; - assert.strictEqual(key.Key, keyName); - assert.strictEqual(key.VersionId, expectedVersionId); - return done(); - }); + makeBackbeatRequest( + { + method: 'GET', + bucket: nullObjectBucket, + queryObj: { 'list-type': 'noncurrent' }, + authCredentials: credentials, + }, + (err, response) => { + assert.ifError(err); + assert.strictEqual(response.statusCode, 200); + const data = JSON.parse(response.body); + + assert.strictEqual(data.IsTruncated, false); + assert(!data.NextKeyMarker); + assert.strictEqual(data.MaxKeys, 1000); + assert.strictEqual(data.Contents.length, 1); + const key = data.Contents[0]; + assert.strictEqual(key.Key, keyName); + assert.strictEqual(key.VersionId, expectedVersionId); + return done(); + }, + ); }); }); diff --git a/tests/functional/backbeat/utils.js b/tests/functional/backbeat/utils.js index 60ef8aa113..3bf40c2f37 100644 --- a/tests/functional/backbeat/utils.js +++ b/tests/functional/backbeat/utils.js @@ -1,7 +1,9 @@ const { makeRequest } = require('../raw-node/utils/makeRequest'); const ipAddress = process.env.IP ? process.env.IP : '127.0.0.1'; -const { models: { ObjectMD } } = require('arsenal'); +const { + models: { ObjectMD }, +} = require('arsenal'); // NOTE: The routes "getMetadata" and "putMetadata" are utilized for modifying the metadata of an object. // This approach is preferred over directly updating the metadata in MongoDB, diff --git a/tests/functional/healthchecks/package.json b/tests/functional/healthchecks/package.json index 6632cf8ef0..034827c5b6 100644 --- a/tests/functional/healthchecks/package.json +++ b/tests/functional/healthchecks/package.json @@ -1,17 +1,16 @@ { - "name": "Test-healthcheck", - "version": "0.1.0", - "description": "Test-healthcheck", - "main": "tests.js", - "private": true, - "repository": "", - "keywords": [ - "test" - ], - "scripts": { - "test": "mocha -t 40000 test/ --exit", - "test-debug": "_mocha -t 40000 test/ --exit" - }, - "author": "" + "name": "Test-healthcheck", + "version": "0.1.0", + "description": "Test-healthcheck", + "main": "tests.js", + "private": true, + "repository": "", + "keywords": [ + "test" + ], + "scripts": { + "test": "mocha -t 40000 test/ --exit", + "test-debug": "_mocha -t 40000 test/ --exit" + }, + "author": "" } - diff --git a/tests/functional/healthchecks/test/checkRoutes.js b/tests/functional/healthchecks/test/checkRoutes.js index 4df9da81b2..0ce8ba92ab 100644 --- a/tests/functional/healthchecks/test/checkRoutes.js +++ b/tests/functional/healthchecks/test/checkRoutes.js @@ -151,8 +151,7 @@ describe('Healthcheck stats', () => { const totalReqs = 5; beforeEach(done => { redis.flushdb(() => { - async.timesSeries(totalReqs, - (n, next) => makeDummyS3Request(next), done); + async.timesSeries(totalReqs, (n, next) => makeDummyS3Request(next), done); }); }); @@ -165,11 +164,9 @@ describe('Healthcheck stats', () => { if (err) { return done(err); } - const expectedStatsRes = { 'requests': totalReqs, '500s': 0, - 'sampleDuration': 30 }; + const expectedStatsRes = { requests: totalReqs, '500s': 0, sampleDuration: 30 }; assert.deepStrictEqual(JSON.parse(res), expectedStatsRes); return done(); }); - }, 500) - ); + }, 500)); }); diff --git a/tests/functional/kmip/serverside_encryption.js b/tests/functional/kmip/serverside_encryption.js index 88badb4733..855b356f06 100644 --- a/tests/functional/kmip/serverside_encryption.js +++ b/tests/functional/kmip/serverside_encryption.js @@ -1,9 +1,4 @@ -const { - S3Client, - PutObjectCommand, - CopyObjectCommand, - CreateMultipartUploadCommand -} = require('@aws-sdk/client-s3'); +const { S3Client, PutObjectCommand, CopyObjectCommand, CreateMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { v4: uuidv4 } = require('uuid'); const config = require('../config.json'); const { auth } = require('arsenal'); @@ -13,7 +8,6 @@ const assert = require('assert'); const logger = { info: msg => process.stdout.write(`${msg}\n`) }; const async = require('async'); - function _createBucket(name, encrypt, done) { const { transport, ipAddress, accessKey, secretKey } = config; const verbose = false; @@ -69,10 +63,8 @@ function _createBucket(name, encrypt, done) { function _buildS3() { const { transport, ipAddress, accessKey, secretKey } = config; - const agent = transport === 'https' - ? new https.Agent({ keepAlive: false }) - : new http.Agent({ keepAlive: false }); - + const agent = transport === 'https' ? new https.Agent({ keepAlive: false }) : new http.Agent({ keepAlive: false }); + return new S3Client({ endpoint: `${transport}://${ipAddress}:8000`, region: 'us-east-1', @@ -107,8 +99,7 @@ function _putObject(bucketName, objectName, encrypt, cb) { .catch(err => cb(err)); } -function _copyObject(sourceBucket, sourceObject, targetBucket, targetObject, - encrypt, cb) { +function _copyObject(sourceBucket, sourceObject, targetBucket, targetObject, encrypt, cb) { const params = { Bucket: targetBucket, CopySource: `/${sourceBucket}/${sourceObject}`, @@ -154,58 +145,62 @@ describe('KMIP backed server-side encryption', () => { it('should create an encrypted bucket', done => { _createBucket(bucketName, true, err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); done(); }); }); it('should create an encrypted bucket and upload an object', done => { - async.waterfall([ - next => _createBucket(bucketName, true, err => next(err)), - next => _putObject(bucketName, objectName, false, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + async.waterfall( + [ + next => _createBucket(bucketName, true, err => next(err)), + next => _putObject(bucketName, objectName, false, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); it('should allow object PUT with SSE header in encrypted bucket', done => { - async.waterfall([ - next => _createBucket(bucketName, true, err => next(err)), - next => _putObject(bucketName, objectName, true, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + async.waterfall( + [ + next => _createBucket(bucketName, true, err => next(err)), + next => _putObject(bucketName, objectName, true, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); it('should allow object copy with SSE header in encrypted bucket', done => { - async.waterfall([ - next => _createBucket(bucketName, false, err => next(err)), - next => _putObject(bucketName, objectName, false, err => next(err)), - next => _createBucket(`${bucketName}2`, true, err => next(err)), - next => _copyObject(bucketName, objectName, `${bucketName}2`, - `${objectName}2`, true, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + async.waterfall( + [ + next => _createBucket(bucketName, false, err => next(err)), + next => _putObject(bucketName, objectName, false, err => next(err)), + next => _createBucket(`${bucketName}2`, true, err => next(err)), + next => _copyObject(bucketName, objectName, `${bucketName}2`, `${objectName}2`, true, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); - it('should allow creating mpu with SSE header ' + - 'in encrypted bucket', done => { - async.waterfall([ - next => _createBucket(bucketName, true, err => next(err)), - next => _initiateMultipartUpload(bucketName, objectName, - true, err => next(err)), - ], err => { - assert.equal(err, null, 'Expected success, ' + - `got error ${JSON.stringify(err)}`); - done(); - }); + it('should allow creating mpu with SSE header ' + 'in encrypted bucket', done => { + async.waterfall( + [ + next => _createBucket(bucketName, true, err => next(err)), + next => _initiateMultipartUpload(bucketName, objectName, true, err => next(err)), + ], + err => { + assert.equal(err, null, 'Expected success, ' + `got error ${JSON.stringify(err)}`); + done(); + }, + ); }); }); diff --git a/tests/functional/metadata/MixedVersionFormat.js b/tests/functional/metadata/MixedVersionFormat.js index a72c708375..b5bf665f5f 100644 --- a/tests/functional/metadata/MixedVersionFormat.js +++ b/tests/functional/metadata/MixedVersionFormat.js @@ -1,11 +1,11 @@ const assert = require('assert'); const async = require('async'); -const { - PutObjectCommand, - GetObjectCommand, - ListObjectsCommand, +const { + PutObjectCommand, + GetObjectCommand, + ListObjectsCommand, PutBucketVersioningCommand, - ListObjectVersionsCommand + ListObjectVersionsCommand, } = require('@aws-sdk/client-s3'); const withV4 = require('../aws-node-sdk/test/support/withV4'); const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); @@ -16,8 +16,8 @@ const replicaSetHosts = 'localhost:27017,localhost:27018,localhost:27019'; const writeConcern = 'majority'; const replicaSet = 'rs0'; const readPreference = 'primary'; -const mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + - `replicaSet=${replicaSet}&readPreference=${readPreference}`; +const mongoUrl = + `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + `replicaSet=${replicaSet}&readPreference=${readPreference}`; /** * These tests are intended to see if the vFormat of buckets is respected @@ -47,61 +47,74 @@ describe('Mongo backend mixed bucket format versions', () => { function updateBucketVFormat(bucketName, vFormat) { const db = mongoClient.db('metadata'); - return db.collection('__metastore') - .updateOne({ + return db.collection('__metastore').updateOne( + { _id: bucketName, - }, { + }, + { $set: { vFormat }, - }, {}); + }, + {}, + ); } function getObject(bucketName, key, cb) { const db = mongoClient.db('metadata'); - return db.collection(bucketName) - .findOne({ - _id: key, - }, {}).then(doc => { - if (!doc) { - return cb(errors.NoSuchKey); - } - return cb(null, doc.value); - }).catch(err => cb(err)); + return db + .collection(bucketName) + .findOne( + { + _id: key, + }, + {}, + ) + .then(doc => { + if (!doc) { + return cb(errors.NoSuchKey); + } + return cb(null, doc.value); + }) + .catch(err => cb(err)); } before(done => { - MongoClient.connect(mongoUrl, {}).then(client => { - mongoClient = client; - bucketUtil = new BucketUtility('default', sigCfg); - s3 = bucketUtil.s3; - return done(); - }).catch(err => done(err)); + MongoClient.connect(mongoUrl, {}) + .then(client => { + mongoClient = client; + bucketUtil = new BucketUtility('default', sigCfg); + s3 = bucketUtil.s3; + return done(); + }) + .catch(err => done(err)); }); beforeEach(() => { process.stdout.write('Creating buckets'); - return bucketUtil.createMany(['v0-bucket', 'v1-bucket']) - .then(async () => { - process.stdout.write('Updating bucket vFormat'); - await updateBucketVFormat('v0-bucket', 'v0'); - await updateBucketVFormat('v1-bucket', 'v1'); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .createMany(['v0-bucket', 'v1-bucket']) + .then(async () => { + process.stdout.write('Updating bucket vFormat'); + await updateBucketVFormat('v0-bucket', 'v0'); + await updateBucketVFormat('v1-bucket', 'v1'); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); afterEach(() => { process.stdout.write('Emptying buckets'); - return bucketUtil.emptyMany(['v0-bucket', 'v1-bucket']) - .then(() => { - process.stdout.write('Deleting buckets'); - return bucketUtil.deleteMany(['v0-bucket', 'v1-bucket']); - }) - .catch(err => { - process.stdout.write('Error in afterEach'); - throw err; - }); + return bucketUtil + .emptyMany(['v0-bucket', 'v1-bucket']) + .then(() => { + process.stdout.write('Deleting buckets'); + return bucketUtil.deleteMany(['v0-bucket', 'v1-bucket']); + }) + .catch(err => { + process.stdout.write('Error in afterEach'); + throw err; + }); }); after(async () => { @@ -112,129 +125,137 @@ describe('Mongo backend mixed bucket format versions', () => { it(`Should perform operations on non versioned bucket in ${vFormat} format`, done => { const paramsObj1 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-1` + Key: `${vFormat}-object-1`, }; const paramsObj2 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-2` + Key: `${vFormat}-object-2`, }; const masterKey = vFormat === 'v0' ? `${vFormat}-object-1` : `\x7fM${vFormat}-object-1`; - async.series([ - next => { - s3.send(new PutObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj2)) - .then(() => next()) - .catch(next); - }, - // check if data stored in the correct format - next => getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { - assert.ifError(err); - assert.strictEqual(doc.key, `${vFormat}-object-1`); - return next(); - }), - // test if we can get object - next => { - s3.send(new GetObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - // test if we can list objects - next => { - s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) - .then(data => { - assert.strictEqual(data.Contents.length, 2); - const keys = data.Contents.map(obj => obj.Key); - assert(keys.includes(`${vFormat}-object-1`)); - assert(keys.includes(`${vFormat}-object-2`)); - next(); - }) - .catch(next); - } - ], done); + async.series( + [ + next => { + s3.send(new PutObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj2)) + .then(() => next()) + .catch(next); + }, + // check if data stored in the correct format + next => + getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { + assert.ifError(err); + assert.strictEqual(doc.key, `${vFormat}-object-1`); + return next(); + }), + // test if we can get object + next => { + s3.send(new GetObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + // test if we can list objects + next => { + s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) + .then(data => { + assert.strictEqual(data.Contents.length, 2); + const keys = data.Contents.map(obj => obj.Key); + assert(keys.includes(`${vFormat}-object-1`)); + assert(keys.includes(`${vFormat}-object-2`)); + next(); + }) + .catch(next); + }, + ], + done, + ); }); it(`Should perform operations on versioned bucket in ${vFormat} format`, done => { const paramsObj1 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-1` + Key: `${vFormat}-object-1`, }; const paramsObj2 = { Bucket: `${vFormat}-bucket`, - Key: `${vFormat}-object-2` + Key: `${vFormat}-object-2`, }; const versioningParams = { Bucket: `${vFormat}-bucket`, VersioningConfiguration: { - Status: 'Enabled', - } + Status: 'Enabled', + }, }; const masterKey = vFormat === 'v0' ? `${vFormat}-object-1` : `\x7fM${vFormat}-object-1`; - - async.series([ - next => { - s3.send(new PutBucketVersioningCommand(versioningParams)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - next => { - s3.send(new PutObjectCommand(paramsObj2)) - .then(() => next()) - .catch(next); - }, - // check if data stored in the correct version format - next => getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { - assert.ifError(err); - assert.strictEqual(doc.key, `${vFormat}-object-1`); - return next(); - }), - // test if we can get object - next => { - s3.send(new GetObjectCommand(paramsObj1)) - .then(() => next()) - .catch(next); - }, - // test if we can list objects - next => { - s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) - .then(data => { - assert.strictEqual(data.Contents.length, 2); - const keys = data.Contents.map(obj => obj.Key); - assert(keys.includes(`${vFormat}-object-1`)); - assert(keys.includes(`${vFormat}-object-2`)); - next(); - }) - .catch(next); - }, - // test if we can list object versions - next => { - s3.send(new ListObjectVersionsCommand({ Bucket: `${vFormat}-bucket` })) - .then(data => { - assert.strictEqual(data.Versions.length, 3); - const versionPerObject = {}; - data.Versions.forEach(version => { - versionPerObject[version.Key] = (versionPerObject[version.Key] || 0) + 1; - }); - assert.strictEqual(versionPerObject[`${vFormat}-object-1`], 2); - assert.strictEqual(versionPerObject[`${vFormat}-object-2`], 1); - next(); - }) - .catch(next); - } - ], done); + + async.series( + [ + next => { + s3.send(new PutBucketVersioningCommand(versioningParams)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + next => { + s3.send(new PutObjectCommand(paramsObj2)) + .then(() => next()) + .catch(next); + }, + // check if data stored in the correct version format + next => + getObject(`${vFormat}-bucket`, masterKey, (err, doc) => { + assert.ifError(err); + assert.strictEqual(doc.key, `${vFormat}-object-1`); + return next(); + }), + // test if we can get object + next => { + s3.send(new GetObjectCommand(paramsObj1)) + .then(() => next()) + .catch(next); + }, + // test if we can list objects + next => { + s3.send(new ListObjectsCommand({ Bucket: `${vFormat}-bucket` })) + .then(data => { + assert.strictEqual(data.Contents.length, 2); + const keys = data.Contents.map(obj => obj.Key); + assert(keys.includes(`${vFormat}-object-1`)); + assert(keys.includes(`${vFormat}-object-2`)); + next(); + }) + .catch(next); + }, + // test if we can list object versions + next => { + s3.send(new ListObjectVersionsCommand({ Bucket: `${vFormat}-bucket` })) + .then(data => { + assert.strictEqual(data.Versions.length, 3); + const versionPerObject = {}; + data.Versions.forEach(version => { + versionPerObject[version.Key] = (versionPerObject[version.Key] || 0) + 1; + }); + assert.strictEqual(versionPerObject[`${vFormat}-object-1`], 2); + assert.strictEqual(versionPerObject[`${vFormat}-object-2`], 1); + next(); + }) + .catch(next); + }, + ], + done, + ); }); }); }); diff --git a/tests/functional/metadata/MongoClientInterface.js b/tests/functional/metadata/MongoClientInterface.js index a8ce1456a2..b883c14986 100644 --- a/tests/functional/metadata/MongoClientInterface.js +++ b/tests/functional/metadata/MongoClientInterface.js @@ -2,9 +2,7 @@ const assert = require('assert'); const async = require('async'); const MongoClient = require('mongodb').MongoClient; -const { - MongoClientInterface, -} = require('arsenal').storage.metadata.mongoclient; +const { MongoClientInterface } = require('arsenal').storage.metadata.mongoclient; const { errors } = require('arsenal'); const log = require('./utils/fakeLogger'); @@ -13,8 +11,8 @@ const replicaSetHosts = 'localhost:27017,localhost:27018,localhost:27019'; const writeConcern = 'majority'; const replicaSet = 'rs0'; const readPreference = 'primary'; -const mongoUrl = `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + - `replicaSet=${replicaSet}&readPreference=${readPreference}`; +const mongoUrl = + `mongodb://${replicaSetHosts}/?w=${writeConcern}&` + `replicaSet=${replicaSet}&readPreference=${readPreference}`; const VID_SEP = '\0'; const TEST_DB = 'test'; @@ -41,13 +39,14 @@ const objVal = { const updatedObjVal = { updated: true }; -const runIfMongo = - process.env.S3METADATA === 'mongodb' ? describe : describe.skip; +const runIfMongo = process.env.S3METADATA === 'mongodb' ? describe : describe.skip; function unescapeJSON(obj) { - return JSON.parse(JSON.stringify(obj). - replace(/\uFF04/g, '$'). - replace(/\uFF0E/g, '.')); + return JSON.parse( + JSON.stringify(obj) + .replace(/\uFF04/g, '$') + .replace(/\uFF0E/g, '.'), + ); } function getCaseMethod(number) { @@ -72,25 +71,28 @@ runIfMongo('MongoClientInterface', () => { if (params && params.versionId) { objName = `${objName}${VID_SEP}${params.versionId}`; } - collection.findOne({ - _id: objName, - }, {}).then(doc => { - if (!doc) { - return cb(errors.NoSuchKey); - } - if (doc.value.tags) { - // eslint-disable-next-line - doc.value.tags = unescapeJSON(doc.value.tags); - } - return cb(null, doc.value); - }).catch(err => cb(err)); + collection + .findOne( + { + _id: objName, + }, + {}, + ) + .then(doc => { + if (!doc) { + return cb(errors.NoSuchKey); + } + if (doc.value.tags) { + // eslint-disable-next-line + doc.value.tags = unescapeJSON(doc.value.tags); + } + return cb(null, doc.value); + }) + .catch(err => cb(err)); } function checkVersionAndMasterMatch(versionId, cb) { - async.parallel([ - next => getObject({}, next), - next => getObject({ versionId }, next), - ], (err, res) => { + async.parallel([next => getObject({}, next), next => getObject({ versionId }, next)], (err, res) => { if (err) { return cb(err); } @@ -104,41 +106,42 @@ runIfMongo('MongoClientInterface', () => { function putObject(caseNum, params, objectValue, cb) { const method = getCaseMethod(caseNum); const dupeObjVal = Object.assign({}, objectValue || objVal); - mongoClientInterface[method](collection, BUCKET_NAME, OBJECT_NAME, - dupeObjVal, params, log, (err, res) => { - if (err) { - return cb(err); - } - let parsedRes; - if (res) { - try { - parsedRes = JSON.parse(res); - } catch (error) { - return cb(error); - } + mongoClientInterface[method](collection, BUCKET_NAME, OBJECT_NAME, dupeObjVal, params, log, (err, res) => { + if (err) { + return cb(err); + } + let parsedRes; + if (res) { + try { + parsedRes = JSON.parse(res); + } catch (error) { + return cb(error); } - return cb(null, parsedRes); - }); + } + return cb(null, parsedRes); + }); } function checkNewPutObject(caseNum, params, cb) { const method = getCaseMethod(caseNum); const bucket = 'a'; const key = 'b'; - async.series([ - next => mongoClientInterface[method]( - collection, bucket, key, updatedObjVal, params, log, next), - next => { - collection.findOne({ _id: key }, (err, result) => { - if (err) { - return next(err); - } - assert.strictEqual(result._id, key); - assert(result.value.updated); - return next(); - }); - }, - ], cb); + async.series( + [ + next => mongoClientInterface[method](collection, bucket, key, updatedObjVal, params, log, next), + next => { + collection.findOne({ _id: key }, (err, result) => { + if (err) { + return next(err); + } + assert.strictEqual(result._id, key); + assert(result.value.updated); + return next(); + }); + }, + ], + cb, + ); } before(done => { @@ -180,11 +183,13 @@ runIfMongo('MongoClientInterface', () => { describe('::putObjectVerCase1', () => { it('should put new metadata and update master', done => { - async.waterfall([ - next => putObject(1, {}, null, - (err, res) => next(err, res.versionId)), - (id, next) => checkVersionAndMasterMatch(id, next), - ], done); + async.waterfall( + [ + next => putObject(1, {}, null, (err, res) => next(err, res.versionId)), + (id, next) => checkVersionAndMasterMatch(id, next), + ], + done, + ); }); }); @@ -192,130 +197,152 @@ runIfMongo('MongoClientInterface', () => { it('should put new metadata', done => checkNewPutObject(2, {}, done)); it('should set new version id for master', done => { - async.waterfall([ - // first create new ver and master - next => putObject(1, {}, null, next), - // check master and version were created and match - (res, next) => checkVersionAndMasterMatch(res.versionId, - err => next(err, res.versionId)), - // call ver case 2 - (id, next) => putObject(2, {}, null, (err, res) => { - if (err) { - return next(err); - } - assert(id !== res.versionId); - return next(null, res.versionId); - }), - // assert master updated with new version id - (newId, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - assert.strictEqual(res.versionId, newId); - return next(null, newId); - }), - // new version entry should not have been created - (id, next) => getObject({ versionId: id }, err => { - assert(err); - assert(err.is.NoSuchKey); - return next(); - }), - ], done); + async.waterfall( + [ + // first create new ver and master + next => putObject(1, {}, null, next), + // check master and version were created and match + (res, next) => checkVersionAndMasterMatch(res.versionId, err => next(err, res.versionId)), + // call ver case 2 + (id, next) => + putObject(2, {}, null, (err, res) => { + if (err) { + return next(err); + } + assert(id !== res.versionId); + return next(null, res.versionId); + }), + // assert master updated with new version id + (newId, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + assert.strictEqual(res.versionId, newId); + return next(null, newId); + }), + // new version entry should not have been created + (id, next) => + getObject({ versionId: id }, err => { + assert(err); + assert(err.is.NoSuchKey); + return next(); + }), + ], + done, + ); }); }); describe('::putObjectVerCase3', () => { - it('should put new metadata', done => - checkNewPutObject(3, { versionId: VERSION_ID }, done)); + it('should put new metadata', done => checkNewPutObject(3, { versionId: VERSION_ID }, done)); it('should put new metadata and not update master', done => { - async.waterfall([ - // first create new ver and master - next => putObject(1, {}, null, next), - // check master and version were created and match - (res, next) => checkVersionAndMasterMatch(res.versionId, - err => next(err, res.versionId)), - // call ver case 3 - (id, next) => putObject(3, { versionId: VERSION_ID }, null, - (err, res) => { - if (err) { - return next(err); - } - // assert new version id created - assert(id !== res.versionId); - assert.strictEqual(res.versionId, VERSION_ID); - return next(null, id); - }), - // assert master did not update and matches old initial version - (oldId, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - assert.strictEqual(oldId, res.versionId); - return next(); - }), - // assert new version was created - next => getObject({ versionId: VERSION_ID }, (err, res) => { - if (err) { - return next(err); - } - assert(res); - assert.strictEqual(res.versionId, VERSION_ID); - return next(); - }), - ], done); + async.waterfall( + [ + // first create new ver and master + next => putObject(1, {}, null, next), + // check master and version were created and match + (res, next) => checkVersionAndMasterMatch(res.versionId, err => next(err, res.versionId)), + // call ver case 3 + (id, next) => + putObject(3, { versionId: VERSION_ID }, null, (err, res) => { + if (err) { + return next(err); + } + // assert new version id created + assert(id !== res.versionId); + assert.strictEqual(res.versionId, VERSION_ID); + return next(null, id); + }), + // assert master did not update and matches old initial version + (oldId, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + assert.strictEqual(oldId, res.versionId); + return next(); + }), + // assert new version was created + next => + getObject({ versionId: VERSION_ID }, (err, res) => { + if (err) { + return next(err); + } + assert(res); + assert.strictEqual(res.versionId, VERSION_ID); + return next(); + }), + ], + done, + ); }); - it('should put new metadata and update master if version id matches', - done => { - async.waterfall([ - // first create new ver and master - next => putObject(1, {}, null, next), - // check master and version were created and match - (res, next) => checkVersionAndMasterMatch(res.versionId, - err => next(err, res.versionId)), - // call ver case 3 w/ same version id and update - (id, next) => mongoClientInterface.putObjectVerCase3(collection, - BUCKET_NAME, OBJECT_NAME, updatedObjVal, - { versionId: id }, log, err => next(err, id)), - (oldId, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - // assert updated - assert(res); - assert(res.updated); - // assert same version id - assert.strictEqual(oldId, res.versionId); - return next(); - }), - ], done); + it('should put new metadata and update master if version id matches', done => { + async.waterfall( + [ + // first create new ver and master + next => putObject(1, {}, null, next), + // check master and version were created and match + (res, next) => checkVersionAndMasterMatch(res.versionId, err => next(err, res.versionId)), + // call ver case 3 w/ same version id and update + (id, next) => + mongoClientInterface.putObjectVerCase3( + collection, + BUCKET_NAME, + OBJECT_NAME, + updatedObjVal, + { versionId: id }, + log, + err => next(err, id), + ), + (oldId, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + // assert updated + assert(res); + assert(res.updated); + // assert same version id + assert.strictEqual(oldId, res.versionId); + return next(); + }), + ], + done, + ); }); }); describe('::putObjectVerCase4', () => { function putAndCheckCase4(versionId, cb) { const objectValue = Object.assign({}, objVal, { versionId }); - async.waterfall([ - // put object - next => putObject(4, { versionId }, objectValue, (err, res) => { - if (err) { - return next(err); - } - return next(null, res.versionId); - }), - (id, next) => getObject({}, (err, res) => { - if (err) { - return next(err); - } - // assert PHD was placed on master - assert(res); - assert.strictEqual(res.isPHD, true); - // assert same version id as master - assert.strictEqual(id, res.versionId); - return next(); - }), - ], cb); + async.waterfall( + [ + // put object + next => + putObject(4, { versionId }, objectValue, (err, res) => { + if (err) { + return next(err); + } + return next(null, res.versionId); + }), + (id, next) => + getObject({}, (err, res) => { + if (err) { + return next(err); + } + // assert PHD was placed on master + assert(res); + assert.strictEqual(res.isPHD, true); + // assert same version id as master + assert.strictEqual(id, res.versionId); + return next(); + }), + ], + cb, + ); } it('should put new metadata and update master', done => { @@ -330,11 +357,14 @@ runIfMongo('MongoClientInterface', () => { const suffix = `22019.${count++}`; return `${prefix}${repID}${suffix}`; } - async.series([ - next => putAndCheckCase4(getNewVersion(), next), - next => putAndCheckCase4(getNewVersion(), next), - next => putAndCheckCase4(getNewVersion(), next), - ], done); + async.series( + [ + next => putAndCheckCase4(getNewVersion(), next), + next => putAndCheckCase4(getNewVersion(), next), + next => putAndCheckCase4(getNewVersion(), next), + ], + done, + ); }); }); }); diff --git a/tests/functional/raw-node/package.json b/tests/functional/raw-node/package.json index 1c20bf3262..4640f5ac23 100644 --- a/tests/functional/raw-node/package.json +++ b/tests/functional/raw-node/package.json @@ -1,23 +1,23 @@ { - "name": "Test-rawnode", - "version": "0.1.0", - "description": "Test-rawnode", - "main": "tests.js", - "private": true, - "repository": "", - "keywords": [ - "test" - ], - "scripts": { - "test-aws": "AWS_ON_AIR=true mocha -t 40000 test/ --exit", - "test-gcp": "mocha -t 40000 test/GCP/ --exit", - "test-routes": "mocha -t 40000 test/routes/ --exit", - "test": "mocha -t 40000 test/ --exit", - "test-debug": "_mocha -t 40000 test/ --exit" - }, - "author": "", - "mocha": { - "recursive": true, - "timeout": 40000 - } + "name": "Test-rawnode", + "version": "0.1.0", + "description": "Test-rawnode", + "main": "tests.js", + "private": true, + "repository": "", + "keywords": [ + "test" + ], + "scripts": { + "test-aws": "AWS_ON_AIR=true mocha -t 40000 test/ --exit", + "test-gcp": "mocha -t 40000 test/GCP/ --exit", + "test-routes": "mocha -t 40000 test/routes/ --exit", + "test": "mocha -t 40000 test/ --exit", + "test-debug": "_mocha -t 40000 test/ --exit" + }, + "author": "", + "mocha": { + "recursive": true, + "timeout": 40000 + } } diff --git a/tests/functional/raw-node/test/GCP/README.MD b/tests/functional/raw-node/test/GCP/README.MD index bfc58a616a..6aecd181ac 100644 --- a/tests/functional/raw-node/test/GCP/README.MD +++ b/tests/functional/raw-node/test/GCP/README.MD @@ -1,5 +1,5 @@ -This directory contains GCP API functional tests. +This directory contains GCP API functional tests. These tests will verify that the GCP API implementation located in `lib/data/external/GCP` behaves as intended: correct error responses, edge case -handling, and correct responses. \ No newline at end of file +handling, and correct responses. diff --git a/tests/functional/raw-node/test/GCP/bucket/bucket.js b/tests/functional/raw-node/test/GCP/bucket/bucket.js index f155a0e0b8..8f33b2c688 100644 --- a/tests/functional/raw-node/test/GCP/bucket/bucket.js +++ b/tests/functional/raw-node/test/GCP/bucket/bucket.js @@ -12,8 +12,7 @@ const { } = require('@aws-sdk/client-s3'); const { GCP } = arsenal.storage.data.external.GCP; const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); const { listingHardLimit } = require('../../../../../../constants'); const credentialOne = 'gcpbackend'; @@ -54,10 +53,7 @@ describe('GCP: Bucket', function testSuite() { const { $metadata, ...data } = res; assert.strictEqual($metadata?.httpStatusCode, 200); // Ensure MetaVersionId is present and non-empty - assert.ok( - typeof data.MetaVersionId === 'string' - && data.MetaVersionId.length > 0 - ); + assert.ok(typeof data.MetaVersionId === 'string' && data.MetaVersionId.length > 0); }); }); @@ -66,81 +62,82 @@ describe('GCP: Bucket', function testSuite() { const bigSize = listingHardLimit + 1; function populateBucket(createdObjects, callback) { - process.stdout.write( - `Putting ${createdObjects.length} objects into bucket\n`); + process.stdout.write(`Putting ${createdObjects.length} objects into bucket\n`); async.mapLimit( createdObjects, 10, - async object => gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: object, - })), + async object => + gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: object, + }), + ), err => { if (err) { process.stdout.write(`err putting objects ${err}\n`); } return callback(err); - } + }, ); } function removeObjects(createdObjects, callback) { - process.stdout.write( - `Deleting ${createdObjects.length} objects from bucket\n`); + process.stdout.write(`Deleting ${createdObjects.length} objects from bucket\n`); async.mapLimit( createdObjects, 10, - async object => gcpClient.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: object, - })), + async object => + gcpClient.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: object, + }), + ), err => { if (err) { process.stdout.write(`err deleting objects ${err}\n`); } return callback(err); - } + }, ); } it('should return 200', async () => { - const res = await gcpClient.send( - new ListObjectsCommand({ Bucket: bucketName })); + const res = await gcpClient.send(new ListObjectsCommand({ Bucket: bucketName })); assert.strictEqual(res.$metadata?.httpStatusCode, 200); }); describe('with less than listingHardLimit number of objects', () => { - const createdObjects = Array.from( - Array(smallSize).keys()).map(i => `someObject-${i}`); + const createdObjects = Array.from(Array(smallSize).keys()).map(i => `someObject-${i}`); before(done => populateBucket(createdObjects, done)); after(done => removeObjects(createdObjects, done)); it(`should list all ${smallSize} created objects`, async () => { - const res = await gcpClient.send( - new ListObjectsCommand({ Bucket: bucketName })); + const res = await gcpClient.send(new ListObjectsCommand({ Bucket: bucketName })); assert.strictEqual(res.Contents.length, smallSize); }); it('should list MaxKeys number of objects with MaxKeys at 10', async () => { - const res = await gcpClient.send(new ListObjectsCommand({ - Bucket: bucketName, - MaxKeys: 10, - })); + const res = await gcpClient.send( + new ListObjectsCommand({ + Bucket: bucketName, + MaxKeys: 10, + }), + ); assert.strictEqual(res.Contents.length, 10); }); }); describe('with more than listingHardLimit number of objects', () => { - const createdObjects = Array.from( - Array(bigSize).keys()).map(i => `someObject-${i}`); + const createdObjects = Array.from(Array(bigSize).keys()).map(i => `someObject-${i}`); before(done => populateBucket(createdObjects, done)); after(done => removeObjects(createdObjects, done)); it('should list at max 1000 of objects created', async () => { - const res = await gcpClient.send( - new ListObjectsCommand({ Bucket: bucketName })); + const res = await gcpClient.send(new ListObjectsCommand({ Bucket: bucketName })); assert.strictEqual(res.Contents.length, listingHardLimit); }); @@ -157,10 +154,12 @@ describe('GCP: Bucket', function testSuite() { // Actual behavior: it returns a list longer than 1000 objects when // max-keys is greater than 1000 it.skip('should list at max 1000, ignoring MaxKeys', async () => { - const res = await gcpClient.send(new ListObjectsCommand({ - Bucket: bucketName, - MaxKeys: 1001, - })); + const res = await gcpClient.send( + new ListObjectsCommand({ + Bucket: bucketName, + MaxKeys: 1001, + }), + ); assert.strictEqual(res.Contents.length, listingHardLimit); }); }); diff --git a/tests/functional/raw-node/test/GCP/bucket/versioning.js b/tests/functional/raw-node/test/GCP/bucket/versioning.js index 45163787a4..1ae5d5afe7 100644 --- a/tests/functional/raw-node/test/GCP/bucket/versioning.js +++ b/tests/functional/raw-node/test/GCP/bucket/versioning.js @@ -8,8 +8,7 @@ const { } = require('@aws-sdk/client-s3'); const { GCP } = arsenal.storage.data.external.GCP; const { genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); const credentialOne = 'gcpbackend'; const config = getRealAwsConfig(credentialOne); @@ -28,24 +27,26 @@ describe('GCP: Bucket Versioning', function testSuite() { }); it('should enable bucket versioning', async () => { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Enabled' }, - })); - - const res = await gcpClient.send( - new GetBucketVersioningCommand({ Bucket: bucketName })); + await gcpClient.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Enabled' }, + }), + ); + + const res = await gcpClient.send(new GetBucketVersioningCommand({ Bucket: bucketName })); assert.strictEqual(res.Status, 'Enabled'); }); it('should disable bucket versioning', async () => { - await gcpClient.send(new PutBucketVersioningCommand({ - Bucket: bucketName, - VersioningConfiguration: { Status: 'Suspended' }, - })); - - const res = await gcpClient.send( - new GetBucketVersioningCommand({ Bucket: bucketName })); + await gcpClient.send( + new PutBucketVersioningCommand({ + Bucket: bucketName, + VersioningConfiguration: { Status: 'Suspended' }, + }), + ); + + const res = await gcpClient.send(new GetBucketVersioningCommand({ Bucket: bucketName })); assert.strictEqual(res.Status, 'Suspended'); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/completeMpu.js b/tests/functional/raw-node/test/GCP/object/completeMpu.js index c839985ffa..d70bf58662 100644 --- a/tests/functional/raw-node/test/GCP/object/completeMpu.js +++ b/tests/functional/raw-node/test/GCP/object/completeMpu.js @@ -4,19 +4,9 @@ const arsenal = require('arsenal'); const { promisify } = require('util'); const { ListObjectsCommand } = require('@aws-sdk/client-s3'); const { GCP, GcpUtils } = arsenal.storage.data.external.GCP; -const { - gcpMpuSetup, - genUniqID, - genBucketName, - gcpRetry, - waitForBucketReady, -} = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { gcpMpuSetup, genUniqID, genBucketName, gcpRetry, waitForBucketReady } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -56,7 +46,8 @@ function listObjectsPaginated(gcpClient, bucketName, cb) { } const command = new ListObjectsCommand(params); - return gcpClient.send(command) + return gcpClient + .send(command) .then(res => { const contents = (res && res.Contents) || []; objects.push(...contents); @@ -67,8 +58,7 @@ function listObjectsPaginated(gcpClient, bucketName, cb) { } // AWS listObjects(V1) pagination: prefer NextMarker, fallback to last key. - marker = (res && res.NextMarker) || - (contents.length ? contents[contents.length - 1].Key : undefined); + marker = (res && res.NextMarker) || (contents.length ? contents[contents.length - 1].Key : undefined); if (!marker) { return cb(null, objects); @@ -87,13 +77,18 @@ function emptyBucket(gcpClient, bucketName, cb) { if (err) { return cb(err); } - return async.eachLimit(objects, 20, (object, next) => { - const deleteParams = { - Bucket: bucketName, - Key: object.Key, - }; - return gcpClient.deleteObject(deleteParams, next); - }, cb); + return async.eachLimit( + objects, + 20, + (object, next) => { + const deleteParams = { + Bucket: bucketName, + Key: object.Key, + }; + return gcpClient.deleteObject(deleteParams, next); + }, + cb, + ); }); } @@ -106,41 +101,39 @@ describe('GCP: Complete MPU', function testSuite() { config = getRealAwsConfig(credentialOne); gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - const cmd = new CreateBucketCommand({ Bucket: bucket.Name }); - await gcpRetry(gcpClient, cmd); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + const cmd = new CreateBucketCommand({ Bucket: bucket.Name }); + await gcpRetry(gcpClient, cmd); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await promisify(emptyBucket)(gcpClient, bucket.Name); - const cmd = new DeleteBucketCommand({ Bucket: bucket.Name }); - await gcpRetry(gcpClient, cmd); - }, - ); + await async.eachSeries(buckets, async bucket => { + await promisify(emptyBucket)(gcpClient, bucket.Name); + const cmd = new DeleteBucketCommand({ Bucket: bucket.Name }); + await gcpRetry(gcpClient, cmd); + }); }); describe('when MPU has 0 parts', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: 0, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: 0, + partSize, + }, + done, + ); }); - it('should return error if 0 parts are given in MPU complete', - function testFn(done) { + it('should return error if 0 parts are given in MPU complete', function testFn(done) { const params = { Bucket: bucketNames.main.Name, MPU: bucketNames.mpu.Name, @@ -159,20 +152,28 @@ describe('GCP: Complete MPU', function testSuite() { describe('when MPU has 1 uploaded part', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: 1, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: 1, + partSize, + }, + done, + ); }); - it('should successfully complete MPU', - function testFn(done) { - const parts = GcpUtils.createMpuList({ - Key: this.test.key, - UploadId: this.test.uploadId, - }, 'parts', 1).map(item => { + it('should successfully complete MPU', function testFn(done) { + const parts = GcpUtils.createMpuList( + { + Key: this.test.key, + UploadId: this.test.uploadId, + }, + 'parts', + 1, + ).map(item => { Object.assign(item, { ETag: this.test.etagList[item.PartNumber - 1], }); @@ -186,8 +187,7 @@ describe('GCP: Complete MPU', function testSuite() { MultipartUpload: { Parts: parts }, }; gcpClient.completeMultipartUpload(params, (err, res) => { - assert.equal(err, null, - `Expected success, but got error ${err}`); + assert.equal(err, null, `Expected success, but got error ${err}`); assert.strictEqual(res.ETag, `"${smallMD5}"`); return done(); }); @@ -197,22 +197,30 @@ describe('GCP: Complete MPU', function testSuite() { describe('when MPU has 1024 uploaded parts', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: numParts, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: numParts, + partSize, + }, + done, + ); }); - it('should successfully complete MPU', - function testFn(done) { + it('should successfully complete MPU', function testFn(done) { this.retries(1); - const parts = GcpUtils.createMpuList({ - Key: this.test.key, - UploadId: this.test.uploadId, - }, 'parts', numParts).map(item => { + const parts = GcpUtils.createMpuList( + { + Key: this.test.key, + UploadId: this.test.uploadId, + }, + 'parts', + numParts, + ).map(item => { Object.assign(item, { ETag: this.test.etagList[item.PartNumber - 1], }); @@ -226,8 +234,7 @@ describe('GCP: Complete MPU', function testSuite() { MultipartUpload: { Parts: parts }, }; gcpClient.completeMultipartUpload(params, (err, res) => { - assert.equal(err, null, - `Expected success, but got error ${err}`); + assert.equal(err, null, `Expected success, but got error ${err}`); assert.strictEqual(res.ETag, `"${bigMD5}"`); return done(); }); diff --git a/tests/functional/raw-node/test/GCP/object/deleteMpu.js b/tests/functional/raw-node/test/GCP/object/deleteMpu.js index e4accccd6c..ef1310a81c 100644 --- a/tests/functional/raw-node/test/GCP/object/deleteMpu.js +++ b/tests/functional/raw-node/test/GCP/object/deleteMpu.js @@ -2,15 +2,9 @@ const assert = require('assert'); const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; -const { gcpMpuSetup, genUniqID, genBucketName, gcpRetry, waitForBucketReady } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, - ListObjectsCommand, -} = require('@aws-sdk/client-s3'); +const { gcpMpuSetup, genUniqID, genBucketName, gcpRetry, waitForBucketReady } = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand, ListObjectsCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -26,8 +20,7 @@ const partSize = 10; function gcpMpuSetupWrapper(params, callback) { gcpMpuSetup(params, (err, result) => { - assert.equal(err, null, - `Unable to setup MPU test, error ${err}`); + assert.equal(err, null, `Unable to setup MPU test, error ${err}`); const { uploadId, etagList } = result; this.currentTest.uploadId = uploadId; this.currentTest.etagList = etagList; @@ -45,122 +38,128 @@ describe('GCP: Abort MPU', function testSuite() { gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucket.Name }), - ); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucket.Name })); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - const listCmd = new ListObjectsCommand({ + await async.eachSeries(buckets, async bucket => { + const listCmd = new ListObjectsCommand({ + Bucket: bucket.Name, + }); + const listRes = await gcpClient.send(listCmd); + await async.map(listRes.Contents || [], async object => { + await gcpClient.deleteObject({ Bucket: bucket.Name, + Key: object.Key, }); - const listRes = await gcpClient.send(listCmd); - await async.map(listRes.Contents || [], async object => { - await gcpClient.deleteObject({ - Bucket: bucket.Name, - Key: object.Key, - }); - }); - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucket.Name }), - ); - }, - ); + }); + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucket.Name })); + }); }); describe('when MPU has 0 parts', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: 0, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: 0, + partSize, + }, + done, + ); }); it('should abort MPU with 0 parts', function testFn(done) { - return async.waterfall([ - next => { - const params = { - Bucket: bucketNames.main.Name, - MPU: bucketNames.mpu.Name, - Key: this.test.key, - UploadId: this.test.uploadId, - }; - gcpClient.abortMultipartUpload(params, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }); - }, - next => { - const keyName = - `${this.test.key}-${this.test.uploadId}/init`; - gcpClient.headObject({ - Bucket: bucketNames.mpu.Name, - Key: keyName, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - return next(); - }); - }, - ], done); + return async.waterfall( + [ + next => { + const params = { + Bucket: bucketNames.main.Name, + MPU: bucketNames.mpu.Name, + Key: this.test.key, + UploadId: this.test.uploadId, + }; + gcpClient.abortMultipartUpload(params, err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }); + }, + next => { + const keyName = `${this.test.key}-${this.test.uploadId}/init`; + gcpClient.headObject( + { + Bucket: bucketNames.mpu.Name, + Key: keyName, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + return next(); + }, + ); + }, + ], + done, + ); }); }); describe('when MPU is incomplete', () => { beforeEach(function beforeFn(done) { this.currentTest.key = `somekey-${genUniqID()}`; - gcpMpuSetupWrapper.call(this, { - gcpClient, - bucketNames, - key: this.currentTest.key, - partCount: numParts, partSize, - }, done); + gcpMpuSetupWrapper.call( + this, + { + gcpClient, + bucketNames, + key: this.currentTest.key, + partCount: numParts, + partSize, + }, + done, + ); }); it('should abort incomplete MPU', function testFn(done) { - return async.waterfall([ - next => { - const params = { - Bucket: bucketNames.main.Name, - MPU: bucketNames.mpu.Name, - Key: this.test.key, - UploadId: this.test.uploadId, - }; - gcpClient.abortMultipartUpload(params, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }); - }, - next => { - const keyName = - `${this.test.key}-${this.test.uploadId}/init`; - gcpClient.headObject({ - Bucket: bucketNames.mpu.Name, - Key: keyName, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - return next(); - }); - }, - ], err => done(err)); + return async.waterfall( + [ + next => { + const params = { + Bucket: bucketNames.main.Name, + MPU: bucketNames.mpu.Name, + Key: this.test.key, + UploadId: this.test.uploadId, + }; + gcpClient.abortMultipartUpload(params, err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }); + }, + next => { + const keyName = `${this.test.key}-${this.test.uploadId}/init`; + gcpClient.headObject( + { + Bucket: bucketNames.mpu.Name, + Key: keyName, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + return next(); + }, + ); + }, + ], + err => done(err), + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/initiateMpu.js b/tests/functional/raw-node/test/GCP/object/initiateMpu.js index 95349f3ade..31587eee7c 100644 --- a/tests/functional/raw-node/test/GCP/object/initiateMpu.js +++ b/tests/functional/raw-node/test/GCP/object/initiateMpu.js @@ -9,12 +9,8 @@ const { gcpCreateMultipartUploadWithRetry, waitForBucketReady, } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { - CreateBucketCommand, - DeleteBucketCommand, -} = require('@aws-sdk/client-s3'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -35,29 +31,17 @@ describe('GCP: Initiate MPU', function testSuite() { config = getRealAwsConfig(credentialOne); gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucket.Name }), - ); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucket.Name })); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucket.Name }), - ); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucket.Name })); + }); }); it('Should create a multipart upload object', async () => { @@ -72,32 +56,37 @@ describe('GCP: Initiate MPU', function testSuite() { const mpuInitKey = `${keyName}-${createRes.UploadId}/init`; const headRes = await new Promise((resolve, reject) => { - gcpClient.headObject({ - Bucket: bucketNames.mpu.Name, - Key: mpuInitKey, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}`); - return reject(err); - } - return resolve(res); - }); + gcpClient.headObject( + { + Bucket: bucketNames.mpu.Name, + Key: mpuInitKey, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return reject(err); + } + return resolve(res); + }, + ); }); assert.strictEqual(headRes.Metadata.special, specialKey); await new Promise((resolve, reject) => { - gcpClient.abortMultipartUpload({ - Bucket: bucketNames.main.Name, - MPU: bucketNames.mpu.Name, - UploadId: createRes.UploadId, - Key: keyName, - }, err => { - if (err) { - return reject(err); - } - return resolve(); - }); + gcpClient.abortMultipartUpload( + { + Bucket: bucketNames.main.Name, + MPU: bucketNames.mpu.Name, + UploadId: createRes.UploadId, + Key: keyName, + }, + err => { + if (err) { + return reject(err); + } + return resolve(); + }, + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/object.js b/tests/functional/raw-node/test/GCP/object/object.js index 5cc3e445b1..e94d400728 100644 --- a/tests/functional/raw-node/test/GCP/object/object.js +++ b/tests/functional/raw-node/test/GCP/object/object.js @@ -3,8 +3,7 @@ const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; const { genUniqID, genBucketName, gcpRetry } = require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); const { CreateBucketCommand, DeleteBucketCommand, @@ -23,10 +22,7 @@ describe('GCP: Object', function testSuite() { const bucketName = genBucketName('object'); before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucketName })); }); after(async () => { @@ -36,10 +32,12 @@ describe('GCP: Object', function testSuite() { async function setupExistingObject(test) { /* eslint-disable no-param-reassign */ test.key = `somekey-${genUniqID()}`; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: test.key, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: test.key, + }), + ); test.uploadId = res.VersionId; test.ETag = res.ETag; /* eslint-enable no-param-reassign */ @@ -49,10 +47,12 @@ describe('GCP: Object', function testSuite() { if (!test.key) { return; } - await gcpClient.send(new DeleteObjectCommand({ - Bucket: bucketName, - Key: test.key, - })); + await gcpClient.send( + new DeleteObjectCommand({ + Bucket: bucketName, + Key: test.key, + }), + ); } describe('HEAD Object', () => { @@ -66,10 +66,12 @@ describe('GCP: Object', function testSuite() { }); it('should successfully retrieve object', async function testFn() { - const res = await gcpClient.send(new HeadObjectCommand({ - Bucket: bucketName, - Key: this.test.key, - })); + const res = await gcpClient.send( + new HeadObjectCommand({ + Bucket: bucketName, + Key: this.test.key, + }), + ); assert.strictEqual(res.ETag, this.test.ETag); assert.ok(res.$metadata && res.$metadata.httpStatusCode === 200); }); @@ -79,14 +81,17 @@ describe('GCP: Object', function testSuite() { it('should return 404', async () => { const badObjectKey = `nonexistingkey-${genUniqID()}`; await new Promise(resolve => { - gcpClient.headObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - resolve(); - }); + gcpClient.headObject( + { + Bucket: bucketName, + Key: badObjectKey, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + resolve(); + }, + ); }); }); }); @@ -103,10 +108,12 @@ describe('GCP: Object', function testSuite() { }); it('should successfully retrieve object', async function testFn() { - const res = await gcpClient.send(new GetObjectCommand({ - Bucket: bucketName, - Key: this.test.key, - })); + const res = await gcpClient.send( + new GetObjectCommand({ + Bucket: bucketName, + Key: this.test.key, + }), + ); assert.strictEqual(res.ETag, this.test.ETag); assert.strictEqual(res.VersionId, this.test.uploadId); }); @@ -115,15 +122,18 @@ describe('GCP: Object', function testSuite() { describe('without existing object in bucket', () => { it('should return 404 and NoSuchKey', done => { const badObjectKey = `nonexistingkey-${genUniqID()}`; - gcpClient.getObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata?.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); + gcpClient.getObject( + { + Bucket: bucketName, + Key: badObjectKey, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata?.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }, + ); }); }); }); @@ -139,35 +149,42 @@ describe('GCP: Object', function testSuite() { }); it('should overwrite object', function testFn(done) { - gcpClient.putObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - assert.notStrictEqual(res.VersionId, this.test.uploadId); - return done(); - }); + gcpClient.putObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (err, res) => { + assert.notStrictEqual(res.VersionId, this.test.uploadId); + return done(); + }, + ); }); }); describe('without existing object in bucket', () => { it('should successfully put object', function testFn(done) { this.test.key = `somekey-${genUniqID()}`; - gcpClient.putObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, putRes) => { - assert.equal(err, null, - `Expected success, got error ${err}`); - gcpClient.getObject({ + gcpClient.putObject( + { Bucket: bucketName, Key: this.test.key, - }, (getErr, getRes) => { - assert.equal(getErr, null, - `Expected success, got error ${getErr}`); - assert.strictEqual(getRes.VersionId, putRes.VersionId); - return done(); - }); - }); + }, + (err, putRes) => { + assert.equal(err, null, `Expected success, got error ${err}`); + gcpClient.getObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (getErr, getRes) => { + assert.equal(getErr, null, `Expected success, got error ${getErr}`); + assert.strictEqual(getRes.VersionId, putRes.VersionId); + return done(); + }, + ); + }, + ); }); }); }); @@ -178,73 +195,88 @@ describe('GCP: Object', function testSuite() { describe('with existing object in bucket', () => { beforeEach(async () => { - await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })); - }); - - it('should successfully delete object', done => { - async.waterfall([ - next => gcpClient.deleteObject({ + await gcpClient.send( + new PutObjectCommand({ Bucket: bucketName, Key: objectKey, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); }), - next => { - gcpClient.send(new GetObjectCommand({ - Bucket: bucketName, - Key: objectKey, - })) - .then(() => { - assert.fail('Expected NoSuchKey error'); - }) - .catch(err => { - assert(err); - assert.strictEqual( - err.$metadata && err.$metadata.httpStatusCode, - 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return next(); - }); - }, - ], err => done(err)); + ); + }); + + it('should successfully delete object', done => { + async.waterfall( + [ + next => + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: objectKey, + }, + err => { + assert.equal(err, null, `Expected success, got error ${err}`); + return next(); + }, + ), + next => { + gcpClient + .send( + new GetObjectCommand({ + Bucket: bucketName, + Key: objectKey, + }), + ) + .then(() => { + assert.fail('Expected NoSuchKey error'); + }) + .catch(err => { + assert(err); + assert.strictEqual(err.$metadata && err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return next(); + }); + }, + ], + err => done(err), + ); }); }); describe('without existing object in bucket', () => { it('should return 404 and NoSuchKey', done => { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: badObjectKey, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: badObjectKey, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }, + ); }); }); }); describe('COPY Object', () => { describe('without existing object in bucket', () => { - it('should return 404 and \'NoSuchKey\'', done => { + it("should return 404 and 'NoSuchKey'", done => { const missingObject = `nonexistingkey-${genUniqID()}`; const someKey = `somekey-${genUniqID()}`; - gcpClient.copyObject({ - Bucket: bucketName, - Key: someKey, - CopySource: `/${bucketName}/${missingObject}`, - }, err => { - assert(err); - assert.strictEqual(err.$metadata.httpStatusCode, 404); - assert.strictEqual(err.name, 'NoSuchKey'); - return done(); - }); + gcpClient.copyObject( + { + Bucket: bucketName, + Key: someKey, + CopySource: `/${bucketName}/${missingObject}`, + }, + err => { + assert(err); + assert.strictEqual(err.$metadata.httpStatusCode, 404); + assert.strictEqual(err.name, 'NoSuchKey'); + return done(); + }, + ); }); }); @@ -253,102 +285,128 @@ describe('GCP: Object', function testSuite() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.copyKey = `copykey-${genUniqID()}`; this.currentTest.initValue = `${genUniqID()}`; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.copyKey, - Metadata: { - value: this.currentTest.initValue, - }, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.copyKey, + Metadata: { + value: this.currentTest.initValue, + }, + }), + ); this.currentTest.ETag = res.ETag; }); afterEach(function afterFn(done) { - async.parallel([ - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}\n`); - } - return next(err); - }), - next => gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.copyKey, - }, err => { - if (err) { - process.stdout - .write(`err in deleting copy object ${err}\n`); - } - return next(err); - }), - ], done); + async.parallel( + [ + next => + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: this.currentTest.key, + }, + err => { + if (err) { + process.stdout.write(`err in deleting object ${err}\n`); + } + return next(err); + }, + ), + next => + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: this.currentTest.copyKey, + }, + err => { + if (err) { + process.stdout.write(`err in deleting copy object ${err}\n`); + } + return next(err); + }, + ), + ], + done, + ); }); - it('should successfully copy with REPLACE directive', - function testFn(done) { + it('should successfully copy with REPLACE directive', function testFn(done) { const newValue = `${genUniqID()}`; - async.waterfall([ - next => gcpClient.copyObject({ - Bucket: bucketName, - Key: this.test.key, - CopySource: `/${bucketName}/${this.test.copyKey}`, - MetadataDirective: 'REPLACE', - Metadata: { - value: newValue, - }, - }, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}\n`); - return next(err); - } - assert.strictEqual(res.ETag, this.test.ETag); - assert.notStrictEqual(res.Metadata.value, - this.test.initValue); - return next(); - }), - ], done); + async.waterfall( + [ + next => + gcpClient.copyObject( + { + Bucket: bucketName, + Key: this.test.key, + CopySource: `/${bucketName}/${this.test.copyKey}`, + MetadataDirective: 'REPLACE', + Metadata: { + value: newValue, + }, + }, + err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }, + ), + next => + gcpClient.headObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}\n`); + return next(err); + } + assert.strictEqual(res.ETag, this.test.ETag); + assert.notStrictEqual(res.Metadata.value, this.test.initValue); + return next(); + }, + ), + ], + done, + ); }); - it('should successfully copy with COPY directive', - function testFn(done) { - async.waterfall([ - next => gcpClient.copyObject({ - Bucket: bucketName, - Key: this.test.key, - CopySource: `/${bucketName}/${this.test.copyKey}`, - MetadataDirective: 'COPY', - }, err => { - assert.equal(err, null, - `Expected success, but got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - }, (err, res) => { - if (err) { - process.stdout - .write(`err in retrieving object ${err}\n`); - return next(err); - } - assert.strictEqual(res.ETag, this.test.ETag); - assert.strictEqual(res.Metadata.value, - this.test.initValue); - return next(); - }), - ], done); + it('should successfully copy with COPY directive', function testFn(done) { + async.waterfall( + [ + next => + gcpClient.copyObject( + { + Bucket: bucketName, + Key: this.test.key, + CopySource: `/${bucketName}/${this.test.copyKey}`, + MetadataDirective: 'COPY', + }, + err => { + assert.equal(err, null, `Expected success, but got error ${err}`); + return next(); + }, + ), + next => + gcpClient.headObject( + { + Bucket: bucketName, + Key: this.test.key, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}\n`); + return next(err); + } + assert.strictEqual(res.ETag, this.test.ETag); + assert.strictEqual(res.Metadata.value, this.test.initValue); + return next(); + }, + ), + ], + done, + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/tagging.js b/tests/functional/raw-node/test/GCP/object/tagging.js index 50f0d4c9e7..000d0c1034 100644 --- a/tests/functional/raw-node/test/GCP/object/tagging.js +++ b/tests/functional/raw-node/test/GCP/object/tagging.js @@ -2,16 +2,17 @@ const assert = require('assert'); const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; -const { genPutTagObj, genGetTagObj, genDelTagObj, genUniqID, genBucketName, gcpRetry } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); -const { gcpTaggingPrefix } = require('../../../../../../constants'); const { - CreateBucketCommand, - DeleteBucketCommand, - PutObjectCommand, -} = require('@aws-sdk/client-s3'); + genPutTagObj, + genGetTagObj, + genDelTagObj, + genUniqID, + genBucketName, + gcpRetry, +} = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { gcpTaggingPrefix } = require('../../../../../../constants'); +const { CreateBucketCommand, DeleteBucketCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; @@ -22,149 +23,162 @@ describe('GCP: Object Tagging', function testSuite() { const bucketName = genBucketName('tagging'); before(async () => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucketName }), - ); + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucketName })); }); after(async () => { - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucketName }), - ); + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucketName })); }); afterEach(function afterFn(done) { - gcpClient.deleteObject({ - Bucket: bucketName, - Key: this.currentTest.key, - }, err => { - if (err) { - process.stdout.write(`err in deleting object ${err}`); - } - return done(err); - }); + gcpClient.deleteObject( + { + Bucket: bucketName, + Key: this.currentTest.key, + }, + err => { + if (err) { + process.stdout.write(`err in deleting object ${err}`); + } + return done(err); + }, + ); }); describe('PUT Object Tagging', () => { beforeEach(async function beforeFn() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + }), + ); this.currentTest.versionId = res.VersionId; }); it('should successfully put object tags', function testFn(done) { - async.waterfall([ - next => gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ + async.waterfall( + [ + next => + gcpClient.putObjectTagging( { - Key: this.test.specialKey, - Value: this.test.specialKey, + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [ + { + Key: this.test.specialKey, + Value: this.test.specialKey, + }, + ], + }, }, - ], - }, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => gcpClient.headObject({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, (err, res) => { - if (err) { - process.stdout.write(`err in retrieving object ${err}`); - return next(err); - } - const metaKey = `${gcpTaggingPrefix}${this.test.specialKey}`; - const toCompare = res.Metadata[metaKey]; - assert.strictEqual(toCompare, this.test.specialKey); - return next(); - }), - ], done); + err => { + assert.equal(err, null, `Expected success, got error ${err}`); + return next(); + }, + ), + next => + gcpClient.headObject( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return next(err); + } + const metaKey = `${gcpTaggingPrefix}${this.test.specialKey}`; + const toCompare = res.Metadata[metaKey]; + assert.strictEqual(toCompare, this.test.specialKey); + return next(); + }, + ), + ], + done, + ); }); describe('when tagging parameter is incorrect', () => { - it('should return 400 and BadRequest if more than ' + - '10 tags are given', function testFun(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: genPutTagObj(11), + it('should return 400 and BadRequest if more than ' + '10 tags are given', function testFun(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: genPutTagObj(11), + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'BadRequest'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'BadRequest'); + return done(); + }, + ); }); - it('should return 400 and InvalidTag if given duplicate keys', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: genPutTagObj(10, true), + it('should return 400 and InvalidTag if given duplicate keys', function testFn(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: genPutTagObj(10, true), + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }, + ); }); - it('should return 400 and InvalidTag if given invalid key', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { Key: Buffer.alloc(129, 'a'), Value: 'bad tag' }, - ], + it('should return 400 and InvalidTag if given invalid key', function testFn(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [{ Key: Buffer.alloc(129, 'a'), Value: 'bad tag' }], + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }, + ); }); - it('should return 400 and InvalidTag if given invalid value', - function testFn(done) { - return gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - Tagging: { - TagSet: [ - { Key: 'badtag', Value: Buffer.alloc(257, 'a') }, - ], + it('should return 400 and InvalidTag if given invalid value', function testFn(done) { + return gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + Tagging: { + TagSet: [{ Key: 'badtag', Value: Buffer.alloc(257, 'a') }], + }, }, - }, err => { - assert(err); - assert.strictEqual(err.code, 400); - assert.strictEqual(err.message, 'InvalidTag'); - return done(); - }); + err => { + assert(err); + assert.strictEqual(err.code, 400); + assert.strictEqual(err.message, 'InvalidTag'); + return done(); + }, + ); }); }); }); @@ -175,82 +189,89 @@ describe('GCP: Object Tagging', function testSuite() { beforeEach(async function beforeFn() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const { expectedTagObj } = - genGetTagObj(tagSize, `x-goog-meta-${gcpTaggingPrefix}`); + const { expectedTagObj } = genGetTagObj(tagSize, `x-goog-meta-${gcpTaggingPrefix}`); this.currentTest.tagObj = expectedTagObj; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + }), + ); this.currentTest.versionId = res.VersionId; await new Promise((resolve, reject) => { - gcpClient.putObjectTagging({ - Bucket: bucketName, - Key: this.currentTest.key, - VersionId: this.currentTest.versionId, - Tagging: { - TagSet: this.currentTest.tagObj, + gcpClient.putObjectTagging( + { + Bucket: bucketName, + Key: this.currentTest.key, + VersionId: this.currentTest.versionId, + Tagging: { + TagSet: this.currentTest.tagObj, + }, }, - }, err => { - if (err) { - process.stdout - .write(`err in setting object tags ${err}`); - reject(err); - return; - } - resolve(); - }); + err => { + if (err) { + process.stdout.write(`err in setting object tags ${err}`); + reject(err); + return; + } + resolve(); + }, + ); }); }); it('should successfully get object tags', function testFn(done) { - gcpClient.getObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, (err, res) => { - assert.equal(err, null, - `Expected success, got error ${err}`); - assert.deepStrictEqual(res.TagSet, this.test.tagObj); - return done(); - }); + gcpClient.getObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, + (err, res) => { + assert.equal(err, null, `Expected success, got error ${err}`); + assert.deepStrictEqual(res.TagSet, this.test.tagObj); + return done(); + }, + ); }); }); describe('DELETE Object Tagging', () => { function assertObjectMetaTag(params, callback) { - return gcpClient.headObject({ - Bucket: params.bucket, - Key: params.key, - VersionId: params.versionId, - }, (err, res) => { - if (err) { - process.stdout.write(`err in retrieving object ${err}`); - return callback(err); - } - const resMeta = Object.assign({}, res.Metadata || {}); - const tagRes = {}; - const metaRes = {}; - Object.keys(resMeta).forEach(key => { - if (key.startsWith(gcpTaggingPrefix)) { - tagRes[key] = resMeta[key]; - } else { - metaRes[key] = resMeta[key]; + return gcpClient.headObject( + { + Bucket: params.bucket, + Key: params.key, + VersionId: params.versionId, + }, + (err, res) => { + if (err) { + process.stdout.write(`err in retrieving object ${err}`); + return callback(err); } - }); - assert.deepStrictEqual(params.tag, tagRes); - assert.deepStrictEqual(params.meta, metaRes); - return callback(); - }); + const resMeta = Object.assign({}, res.Metadata || {}); + const tagRes = {}; + const metaRes = {}; + Object.keys(resMeta).forEach(key => { + if (key.startsWith(gcpTaggingPrefix)) { + tagRes[key] = resMeta[key]; + } else { + metaRes[key] = resMeta[key]; + } + }); + assert.deepStrictEqual(params.tag, tagRes); + assert.deepStrictEqual(params.meta, metaRes); + return callback(); + }, + ); } beforeEach(async function beforeFn() { this.currentTest.key = `somekey-${genUniqID()}`; this.currentTest.specialKey = `veryspecial-${genUniqID()}`; - const { expectedTagObj, expectedMetaObj } = - genDelTagObj(10, `x-goog-meta-${gcpTaggingPrefix}`); + const { expectedTagObj, expectedMetaObj } = genDelTagObj(10, `x-goog-meta-${gcpTaggingPrefix}`); const expectedTagMeta = {}; Object.keys(expectedTagObj).forEach(header => { @@ -267,40 +288,56 @@ describe('GCP: Object Tagging', function testSuite() { this.currentTest.expectedTagObj = expectedTagMeta; this.currentTest.expectedMetaObj = expectedMetaMeta; - const res = await gcpClient.send(new PutObjectCommand({ - Bucket: bucketName, - Key: this.currentTest.key, - Metadata: Object.assign({}, expectedTagMeta, expectedMetaMeta), - })); + const res = await gcpClient.send( + new PutObjectCommand({ + Bucket: bucketName, + Key: this.currentTest.key, + Metadata: Object.assign({}, expectedTagMeta, expectedMetaMeta), + }), + ); this.currentTest.versionId = res.VersionId; }); it('should successfully delete object tags', function testFn(done) { - async.waterfall([ - next => assertObjectMetaTag({ - bucket: bucketName, - key: this.test.key, - versionId: this.test.versionId, - meta: this.test.expectedMetaObj, - tag: this.test.expectedTagObj, - }, next), - next => gcpClient.deleteObjectTagging({ - Bucket: bucketName, - Key: this.test.key, - VersionId: this.test.versionId, - }, err => { - assert.equal(err, null, - `Expected success, got error ${err}`); - return next(); - }), - next => assertObjectMetaTag({ - bucket: bucketName, - key: this.test.key, - versionId: this.test.versionId, - meta: this.test.expectedMetaObj, - tag: {}, - }, next), - ], done); + async.waterfall( + [ + next => + assertObjectMetaTag( + { + bucket: bucketName, + key: this.test.key, + versionId: this.test.versionId, + meta: this.test.expectedMetaObj, + tag: this.test.expectedTagObj, + }, + next, + ), + next => + gcpClient.deleteObjectTagging( + { + Bucket: bucketName, + Key: this.test.key, + VersionId: this.test.versionId, + }, + err => { + assert.equal(err, null, `Expected success, got error ${err}`); + return next(); + }, + ), + next => + assertObjectMetaTag( + { + bucket: bucketName, + key: this.test.key, + versionId: this.test.versionId, + meta: this.test.expectedMetaObj, + tag: {}, + }, + next, + ), + ], + done, + ); }); }); }); diff --git a/tests/functional/raw-node/test/GCP/object/upload.js b/tests/functional/raw-node/test/GCP/object/upload.js index 8b4445a296..f9ac187028 100644 --- a/tests/functional/raw-node/test/GCP/object/upload.js +++ b/tests/functional/raw-node/test/GCP/object/upload.js @@ -2,15 +2,15 @@ const assert = require('assert'); const async = require('async'); const arsenal = require('arsenal'); const { GCP } = arsenal.storage.data.external.GCP; -const { genUniqID, genBucketName, gcpRetry, gcpUploadWithRetry, waitForBucketReady } = - require('../../../utils/gcpUtils'); -const { getRealAwsConfig } = - require('../../../../aws-node-sdk/test/support/awsConfig'); const { - CreateBucketCommand, - DeleteBucketCommand, - ListObjectsCommand, -} = require('@aws-sdk/client-s3'); + genUniqID, + genBucketName, + gcpRetry, + gcpUploadWithRetry, + waitForBucketReady, +} = require('../../../utils/gcpUtils'); +const { getRealAwsConfig } = require('../../../../aws-node-sdk/test/support/awsConfig'); +const { CreateBucketCommand, DeleteBucketCommand, ListObjectsCommand } = require('@aws-sdk/client-s3'); const credentialOne = 'gcpbackend'; const bucketNames = { @@ -36,39 +36,27 @@ describe('GCP: Upload Object', function testSuite() { config = getRealAwsConfig(credentialOne); gcpClient = new GCP(config); const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - await gcpRetry( - gcpClient, - new CreateBucketCommand({ Bucket: bucket.Name }), - ); - await waitForBucketReady(gcpClient, bucket.Name); - }, - ); + await async.eachSeries(buckets, async bucket => { + await gcpRetry(gcpClient, new CreateBucketCommand({ Bucket: bucket.Name })); + await waitForBucketReady(gcpClient, bucket.Name); + }); }); after(async () => { const buckets = Object.values(bucketNames); - await async.eachSeries( - buckets, - async bucket => { - const listCmd = new ListObjectsCommand({ + await async.eachSeries(buckets, async bucket => { + const listCmd = new ListObjectsCommand({ + Bucket: bucket.Name, + }); + const listRes = await gcpClient.send(listCmd); + await async.map(listRes.Contents || [], async object => { + await gcpClient.deleteObject({ Bucket: bucket.Name, + Key: object.Key, }); - const listRes = await gcpClient.send(listCmd); - await async.map(listRes.Contents || [], async object => { - await gcpClient.deleteObject({ - Bucket: bucket.Name, - Key: object.Key, - }); - }); - await gcpRetry( - gcpClient, - new DeleteBucketCommand({ Bucket: bucket.Name }), - ); - }, - ); + }); + await gcpRetry(gcpClient, new DeleteBucketCommand({ Bucket: bucket.Name })); + }); }); it('should put an object to GCP', async () => { diff --git a/tests/functional/raw-node/test/badChunkSignatureV4.js b/tests/functional/raw-node/test/badChunkSignatureV4.js index 4a05e2d7d7..9a5b03ec0a 100644 --- a/tests/functional/raw-node/test/badChunkSignatureV4.js +++ b/tests/functional/raw-node/test/badChunkSignatureV4.js @@ -2,14 +2,12 @@ const http = require('http'); const async = require('async'); const assert = require('assert'); -const BucketUtility = - require('../../aws-node-sdk/lib/utility/bucket-util'); +const BucketUtility = require('../../aws-node-sdk/lib/utility/bucket-util'); const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const config = require('../../config.json'); -const DUMMY_SIGNATURE = - 'baadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0de'; +const DUMMY_SIGNATURE = 'baadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0debaadc0de'; http.globalAgent.keepAlive = true; @@ -31,10 +29,7 @@ function createBucket(bucketUtil, cb) { function cleanupBucket(bucketUtil, cb) { const emptyBucket = async.asyncify(bucketUtil.empty.bind(bucketUtil)); const deleteBucket = async.asyncify(bucketUtil.deleteOne.bind(bucketUtil)); - async.series([ - done => emptyBucket(BUCKET, done), - done => deleteBucket(BUCKET, done), - ], cb); + async.series([done => emptyBucket(BUCKET, done), done => deleteBucket(BUCKET, done)], cb); } class HttpChunkedUploadWithBadSignature extends HttpRequestAuthV4 { @@ -60,39 +55,45 @@ class HttpChunkedUploadWithBadSignature extends HttpRequestAuthV4 { function testChunkedPutWithBadSignature(n, alterSignatureChunkId, cb) { const req = new HttpChunkedUploadWithBadSignature( - `http://${config.ipAddress}:${PORT}/${BUCKET}/obj-${n}`, { + `http://${config.ipAddress}:${PORT}/${BUCKET}/obj-${n}`, + { accessKey: config.accessKey, secretKey: config.secretKey, method: 'PUT', headers: { 'content-length': N_DATA_CHUNKS * DATA_CHUNK_SIZE, - 'connection': 'keep-alive', + connection: 'keep-alive', }, alterSignatureChunkId, - }, res => { - if (alterSignatureChunkId >= 0 && - alterSignatureChunkId <= N_DATA_CHUNKS) { + }, + res => { + if (alterSignatureChunkId >= 0 && alterSignatureChunkId <= N_DATA_CHUNKS) { assert.strictEqual(res.statusCode, 403); } else { assert.strictEqual(res.statusCode, 200); } res.on('data', () => {}); res.on('end', cb); - }); + }, + ); req.on('error', err => { assert.ifError(err); }); - async.timesSeries(N_DATA_CHUNKS, (chunkIndex, done) => { - // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`); - if (req.write(CHUNK_DATA)) { - process.nextTick(done); - } else { - req.once('drain', done); - } - }, () => { - req.end(); - }); + async.timesSeries( + N_DATA_CHUNKS, + (chunkIndex, done) => { + // console.log(`SENDING NEXT CHUNK OF LENGTH ${CHUNK_DATA.length}`); + if (req.write(CHUNK_DATA)) { + process.nextTick(done); + } else { + req.once('drain', done); + } + }, + () => { + req.end(); + }, + ); } describe('streaming V4 signature with bad chunk signature', () => { @@ -100,26 +101,32 @@ describe('streaming V4 signature with bad chunk signature', () => { before(done => createBucket(bucketUtil, done)); after(done => cleanupBucket(bucketUtil, done)); - it('Cloudserver should be robust against bad signature in streaming ' + - 'payload', function badSignatureInStreamingPayload(cb) { - this.timeout(120000); - async.timesLimit(N_PUTS, 10, (n, done) => { - // multiple test cases depend on the value of - // alterSignatureChunkId: - // alterSignatureChunkId >= 0 && - // alterSignatureChunkId < N_DATA_CHUNKS - // <=> alter the signature of the target data chunk - // alterSignatureChunkId == N_DATA_CHUNKS - // <=> alter the signature of the last empty chunk that - // carries the last payload signature - // alterSignatureChunkId > N_DATA_CHUNKS - // <=> no signature is altered (regular test case) - // By making n go from 0 to nDatachunks+1, we cover all - // above cases. - - const alterSignatureChunkId = ALTER_CHUNK_SIGNATURE ? - (n % (N_DATA_CHUNKS + 2)) : null; - testChunkedPutWithBadSignature(n, alterSignatureChunkId, done); - }, err => cb(err)); - }); + it( + 'Cloudserver should be robust against bad signature in streaming ' + 'payload', + function badSignatureInStreamingPayload(cb) { + this.timeout(120000); + async.timesLimit( + N_PUTS, + 10, + (n, done) => { + // multiple test cases depend on the value of + // alterSignatureChunkId: + // alterSignatureChunkId >= 0 && + // alterSignatureChunkId < N_DATA_CHUNKS + // <=> alter the signature of the target data chunk + // alterSignatureChunkId == N_DATA_CHUNKS + // <=> alter the signature of the last empty chunk that + // carries the last payload signature + // alterSignatureChunkId > N_DATA_CHUNKS + // <=> no signature is altered (regular test case) + // By making n go from 0 to nDatachunks+1, we cover all + // above cases. + + const alterSignatureChunkId = ALTER_CHUNK_SIGNATURE ? n % (N_DATA_CHUNKS + 2) : null; + testChunkedPutWithBadSignature(n, alterSignatureChunkId, done); + }, + err => cb(err), + ); + }, + ); }); diff --git a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js index 165d9406f1..c4502f41c2 100644 --- a/tests/functional/raw-node/test/checksumPutObjectUploadPart.js +++ b/tests/functional/raw-node/test/checksumPutObjectUploadPart.js @@ -637,7 +637,6 @@ function makeScenarioTests(urlFn, { expectsImplicitChecksum = true } = {}) { itSkipIfAWS('should return 200 for trailer line with whitespace around name and value', done => { // TrailingChecksumTransform trims both name and value, so whitespace is accepted. - // eslint-disable-next-line max-len -- prettier keeps this fixture template on one line (121 > 120) const body = `f\r\ntrailer content\r\n0\r\n x-amz-checksum-sha256 : ${trailerContentSha256} \n\r\n\r\n\r\n`; doPutRequest( urlFn(), diff --git a/tests/functional/raw-node/test/headObject.js b/tests/functional/raw-node/test/headObject.js index c691dafd97..d79d6fc04b 100644 --- a/tests/functional/raw-node/test/headObject.js +++ b/tests/functional/raw-node/test/headObject.js @@ -10,36 +10,45 @@ const bucket = 'rawnodeapibucket'; describe('api tests', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); it('should return 405 on headBucket when bucket is empty string', done => { - makeS3Request({ - method: 'HEAD', - authCredentials, - bucket: '', - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 405); - return done(); - }); + makeS3Request( + { + method: 'HEAD', + authCredentials, + bucket: '', + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 405); + return done(); + }, + ); }); }); diff --git a/tests/functional/raw-node/test/lifecycle.js b/tests/functional/raw-node/test/lifecycle.js index af145845bb..6d05669364 100644 --- a/tests/functional/raw-node/test/lifecycle.js +++ b/tests/functional/raw-node/test/lifecycle.js @@ -25,109 +25,133 @@ function makeLifeCycleXML(date) { describe('api tests', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); it('should accept a lifecycle policy with a date at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00Z'), - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00Z'), + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + return done(); + }, + ); }); it('should accept a lifecycle policy with a date at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00'), - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00'), + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + return done(); + }, + ); }); it('should accept a lifecycle policy with a date at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T06:00:00+06:00'), - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T06:00:00+06:00'), + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + return done(); + }, + ); }); it('should reject a lifecycle policy with a date not at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T12:34:56Z'), - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - assert.strictEqual(err.statusCode, 400); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T12:34:56Z'), + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + assert.strictEqual(err.statusCode, 400); + return done(); + }, + ); }); it('should reject a lifecycle policy with an illegal date', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00+34:00'), - }, err => { - // This value is catched by AWS during XML parsing - assert(err.code === 'InvalidArgument' || err.code === 'MalformedXML'); - assert.strictEqual(err.statusCode, 400); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00+34:00'), + }, + err => { + // This value is catched by AWS during XML parsing + assert(err.code === 'InvalidArgument' || err.code === 'MalformedXML'); + assert.strictEqual(err.statusCode, 400); + return done(); + }, + ); }); it('should reject a lifecycle policy with a date not at midnight', done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - queryObj: { lifecycle: '' }, - requestBody: makeLifeCycleXML('2024-01-08T00:00:00.123Z'), - }, err => { - assert.strictEqual(err.code, 'InvalidArgument'); - assert.strictEqual(err.statusCode, 400); - return done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + queryObj: { lifecycle: '' }, + requestBody: makeLifeCycleXML('2024-01-08T00:00:00.123Z'), + }, + err => { + assert.strictEqual(err.code, 'InvalidArgument'); + assert.strictEqual(err.statusCode, 400); + return done(); + }, + ); }); }); diff --git a/tests/functional/raw-node/test/malformedDateHeader.js b/tests/functional/raw-node/test/malformedDateHeader.js index eb9a1bcedc..256b5a5985 100644 --- a/tests/functional/raw-node/test/malformedDateHeader.js +++ b/tests/functional/raw-node/test/malformedDateHeader.js @@ -12,12 +12,13 @@ describe('malformed Date header:', () => { path: `/${bucket}/${objectKey}`, method: 'GET', headers: { - 'Date': 'BAD_DATE', - 'Authorization': 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + + Date: 'BAD_DATE', + Authorization: + 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host, Signature=d459d5b2a2395b4c65d8f8aa2729b22c5abb04614fafbd93ab4fe203e76d21a3', 'X-Amz-Content-Sha256': 'fa8d015f89da2a769d1cea7e3bd77a5670d098d7844cda148a40c1304e5b778b', - 'Host': 'localhost:8000' - } + Host: 'localhost:8000', + }, }; const req = http.request(options, res => { @@ -49,12 +50,13 @@ describe('malformed Date header:', () => { method: 'GET', headers: { 'X-Amz-Date': 'BAD_DATE', - 'Authorization': 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + + Authorization: + 'AWS4-HMAC-SHA256 Credential=accessKey1/20260211/us-east-1/s3/aws4_request, ' + 'SignedHeaders=host;x-amz-date, ' + 'Signature=d459d5b2a2395b4c65d8f8aa2729b22c5abb04614fafbd93ab4fe203e76d21a3', 'X-Amz-Content-Sha256': 'fa8d015f89da2a769d1cea7e3bd77a5670d098d7844cda148a40c1304e5b778b', - 'Host': 'localhost:8000' - } + Host: 'localhost:8000', + }, }; const req = http.request(options, res => { diff --git a/tests/functional/raw-node/test/routes/routeMetadata.js b/tests/functional/raw-node/test/routes/routeMetadata.js index a27345b905..d30b1bd14b 100644 --- a/tests/functional/raw-node/test/routes/routeMetadata.js +++ b/tests/functional/raw-node/test/routes/routeMetadata.js @@ -1,10 +1,12 @@ const assert = require('assert'); const crypto = require('crypto'); const http = require('http'); -const { CreateBucketCommand, +const { + CreateBucketCommand, PutObjectCommand, DeleteObjectCommand, - DeleteBucketCommand } = require('@aws-sdk/client-s3'); + DeleteBucketCommand, +} = require('@aws-sdk/client-s3'); const { makeRequest } = require('../../utils/makeRequest'); const MetadataMock = require('../../utils/MetadataMock'); @@ -22,8 +24,7 @@ const metadataAuthCredentials = { }; function makeMetadataRequest(params, callback) { - const { method, headers, authCredentials, - requestBody, queryObj, path } = params; + const { method, headers, authCredentials, requestBody, queryObj, path } = params; const options = { authCredentials, hostname: ipAddress, @@ -39,8 +40,7 @@ function makeMetadataRequest(params, callback) { } describe('metadata routes with metadata', () => { - const bucketUtil = new BucketUtility( - 'default', { signatureVersion: 'v4' }); + const bucketUtil = new BucketUtility('default', { signatureVersion: 'v4' }); const s3 = bucketUtil.s3; const bucket1 = 'bucket1'; @@ -65,85 +65,99 @@ describe('metadata routes with metadata', () => { let httpServer; before(done => { - httpServer = http.createServer( - (req, res) => metadataMock.onRequest(req, res)).listen(9000, done); + httpServer = http.createServer((req, res) => metadataMock.onRequest(req, res)).listen(9000, done); }); after(() => httpServer.close()); } it('should retrieve list of buckets', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: '/_/metadata/admin/raft_sessions/1/bucket', - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - assert(res.body); - const expectedArray = [bucket1, 'users..bucket', bucket2]; - const responseArray = JSON.parse(res.body); - - expectedArray.sort(); - responseArray.sort(); - - assert.deepStrictEqual(responseArray, expectedArray); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: '/_/metadata/admin/raft_sessions/1/bucket', + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + assert(res.body); + const expectedArray = [bucket1, 'users..bucket', bucket2]; + const responseArray = JSON.parse(res.body); + + expectedArray.sort(); + responseArray.sort(); + + assert.deepStrictEqual(responseArray, expectedArray); + return done(); + }, + ); }); it('should retrieve list of objects from bucket', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/bucket/${bucket1}`, - queryObj: { listingType: 'Delimiter' }, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - const body = JSON.parse(res.body); - assert.strictEqual(body.Contents[0].key, 'testobject1'); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket1}`, + queryObj: { listingType: 'Delimiter' }, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const body = JSON.parse(res.body); + assert.strictEqual(body.Contents[0].key, 'testobject1'); + return done(); + }, + ); }); it('should retrieve metadata of bucket', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/attributes/${bucket1}`, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - assert(res.body); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/attributes/${bucket1}`, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + assert(res.body); + return done(); + }, + ); }); it('should retrieve metadata of object', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/bucket/${bucket1}/${keyName}`, - }, (err, res) => { - assert.ifError(err); - assert(res.body); - assert.strictEqual(res.statusCode, 200); - const body = JSON.parse(res.body); - assert(body['owner-id']); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket1}/${keyName}`, + }, + (err, res) => { + assert.ifError(err); + assert(res.body); + assert.strictEqual(res.statusCode, 200); + const body = JSON.parse(res.body); + assert(body['owner-id']); + return done(); + }, + ); }); it('should get an error for accessing invalid routes', done => { - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: '/_/metadata/admin/raft_sessions', - }, err => { - assert.strictEqual(err.code, 'NotImplemented'); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: '/_/metadata/admin/raft_sessions', + }, + err => { + assert.strictEqual(err.code, 'NotImplemented'); + return done(); + }, + ); }); }); @@ -166,17 +180,21 @@ describe('checksum stored in object metadata after PutObject', () => { await s3.send(new CreateBucketCommand({ Bucket: bucket })); const sha256Value = crypto.createHash('sha256').update(objectBody).digest('base64'); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: sha256Key, - Body: objectBody, - ChecksumSHA256: sha256Value, - })); - await s3.send(new PutObjectCommand({ - Bucket: bucket, - Key: defaultKey, - Body: objectBody, - })); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: sha256Key, + Body: objectBody, + ChecksumSHA256: sha256Value, + }), + ); + await s3.send( + new PutObjectCommand({ + Bucket: bucket, + Key: defaultKey, + Body: objectBody, + }), + ); }); after(async () => { @@ -190,28 +208,32 @@ describe('checksum stored in object metadata after PutObject', () => { it('should store sha256 checksum in metadata when x-amz-checksum-sha256 is provided', done => { const expectedValue = crypto.createHash('sha256').update(objectBody).digest('base64'); - makeMetadataRequest({ - method: 'GET', - authCredentials: metadataAuthCredentials, - path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - const md = JSON.parse(res.body); - assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); - assert.strictEqual(md.checksum.checksumValue, expectedValue); - assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); - return done(); - }); + makeMetadataRequest( + { + method: 'GET', + authCredentials: metadataAuthCredentials, + path: `/_/metadata/default/bucket/${bucket}/${sha256Key}`, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + const md = JSON.parse(res.body); + assert.strictEqual(md.checksum.checksumAlgorithm, 'sha256'); + assert.strictEqual(md.checksum.checksumValue, expectedValue); + assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); + return done(); + }, + ); }); - it('should store crc32 checksum in metadata when no explicit checksum header is provided (AWS SDK default)', - done => { - makeMetadataRequest({ + it('should store crc32 checksum in metadata when no explicit checksum header is provided (AWS SDK default)', done => { + makeMetadataRequest( + { method: 'GET', authCredentials: metadataAuthCredentials, path: `/_/metadata/default/bucket/${bucket}/${defaultKey}`, - }, (err, res) => { + }, + (err, res) => { assert.ifError(err); assert.strictEqual(res.statusCode, 200); const md = JSON.parse(res.body); @@ -219,6 +241,7 @@ describe('checksum stored in object metadata after PutObject', () => { assert.strictEqual(md.checksum.checksumValue, expectedCrc32); assert.strictEqual(md.checksum.checksumType, 'FULL_OBJECT'); return done(); - }); - }); + }, + ); + }); }); diff --git a/tests/functional/raw-node/test/trailingChecksums.js b/tests/functional/raw-node/test/trailingChecksums.js index 925d959a7f..45237c2680 100644 --- a/tests/functional/raw-node/test/trailingChecksums.js +++ b/tests/functional/raw-node/test/trailingChecksums.js @@ -6,9 +6,8 @@ const HttpRequestAuthV4 = require('../utils/HttpRequestAuthV4'); const bucket = 'testunsupportedchecksumsbucket'; const objectKey = 'key'; const objData = Buffer.alloc(1024, 'a'); -const objDataWithTrailingChecksum = '10\r\n0123456789abcdef\r\n' + - '10\r\n0123456789abcdef\r\n' + - '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n'; +const objDataWithTrailingChecksum = + '10\r\n0123456789abcdef\r\n' + '10\r\n0123456789abcdef\r\n' + '0\r\nx-amz-checksum-crc64nvme:skQv82y5rgE=\r\n'; const objDataWithoutTrailingChecksum = '0123456789abcdef0123456789abcdef'; const config = require('../../config.json'); @@ -21,33 +20,47 @@ const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; describe('trailing checksum requests:', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - }, next), - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + }, + next, + ), + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + next, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should accept unsigned trailing checksum', done => { @@ -63,13 +76,13 @@ describe('trailing checksum requests:', () => { 'x-amz-trailer': 'x-amz-checksum-crc64nvme', }, }, - authCredentials + authCredentials, ), res => { assert.strictEqual(res.statusCode, 200); res.on('data', () => {}); res.on('end', done); - } + }, ); req.on('error', err => { @@ -84,18 +97,21 @@ describe('trailing checksum requests:', () => { }); it('should have correct object content for unsigned trailing checksum', done => { - makeS3Request({ - method: 'GET', - authCredentials, - bucket, - objectKey, - }, (err, res) => { - assert.ifError(err); - assert.strictEqual(res.statusCode, 200); - // check that the object data is the input stripped of the trailing checksum - assert.strictEqual(res.body, objDataWithoutTrailingChecksum); - return done(); - }); + makeS3Request( + { + method: 'GET', + authCredentials, + bucket, + objectKey, + }, + (err, res) => { + assert.ifError(err); + assert.strictEqual(res.statusCode, 200); + // check that the object data is the input stripped of the trailing checksum + assert.strictEqual(res.body, objDataWithoutTrailingChecksum); + return done(); + }, + ); }); itSkipIfAWS('should respond with BadRequest for signed trailing checksum', done => { @@ -110,13 +126,13 @@ describe('trailing checksum requests:', () => { 'x-amz-trailer': 'x-amz-checksum-sha256', }, }, - authCredentials + authCredentials, ), res => { assert.strictEqual(res.statusCode, 400); res.on('data', () => {}); res.on('end', done); - } + }, ); req.on('error', err => { diff --git a/tests/functional/raw-node/test/unsignedChecksumHeaders.js b/tests/functional/raw-node/test/unsignedChecksumHeaders.js index 26340ff496..87b422891f 100644 --- a/tests/functional/raw-node/test/unsignedChecksumHeaders.js +++ b/tests/functional/raw-node/test/unsignedChecksumHeaders.js @@ -34,7 +34,7 @@ class HttpRequestAuthV4NoSHA256SignedHeader extends HttpRequestAuthV4 { const urlObj = new url.URL(this._url); const signedHeaders = { - 'host': urlObj.host, + host: urlObj.host, 'x-amz-date': this._timestamp, }; const httpHeaders = Object.assign({}, this._httpParams.headers); @@ -44,48 +44,63 @@ class HttpRequestAuthV4NoSHA256SignedHeader extends HttpRequestAuthV4 { signedHeaders[lowerHeader] = httpHeaders[header]; } }); - httpHeaders.Authorization = - this.getAuthorizationHeader(urlObj, signedHeaders, httpHeaders['x-amz-content-sha256']); + httpHeaders.Authorization = this.getAuthorizationHeader( + urlObj, + signedHeaders, + httpHeaders['x-amz-content-sha256'], + ); return Object.assign(httpHeaders, signedHeaders); } } describe('unsigned x-amz-content-sha256 header in AuthV4 requests:', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - async.series([ - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - objectKey, - }, next), - next => makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, next), - ], err => { - assert.ifError(err); - done(); - }); + async.series( + [ + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + objectKey, + }, + next, + ), + next => + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + next, + ), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); it('should accept x-amz-content-sha256 header not in SignedHeaders list', done => { // Calculate the SHA256 hash of the data - const contentSha256 = crypto.createHash('sha256') - .update(objData) - .digest('hex'); + const contentSha256 = crypto.createHash('sha256').update(objData).digest('hex'); const req = new HttpRequestAuthV4NoSHA256SignedHeader( `http://localhost:8000/${bucket}/${objectKey}`, @@ -97,7 +112,7 @@ describe('unsigned x-amz-content-sha256 header in AuthV4 requests:', () => { 'x-amz-content-sha256': contentSha256, }, }, - authCredentials + authCredentials, ), res => { let body = ''; @@ -106,11 +121,14 @@ describe('unsigned x-amz-content-sha256 header in AuthV4 requests:', () => { }); res.on('end', () => { assert.strictEqual(body, '', 'expected empty body'); - assert.strictEqual(res.statusCode, 200, - 'Request should succeed even when x-amz-content-sha256 is not signed'); + assert.strictEqual( + res.statusCode, + 200, + 'Request should succeed even when x-amz-content-sha256 is not signed', + ); done(); }); - } + }, ); req.on('error', err => { diff --git a/tests/functional/raw-node/test/unsupportedChecksums.js b/tests/functional/raw-node/test/unsupportedChecksums.js index dd04be0f5f..ed3ac96ecf 100644 --- a/tests/functional/raw-node/test/unsupportedChecksums.js +++ b/tests/functional/raw-node/test/unsupportedChecksums.js @@ -15,25 +15,31 @@ const itSkipIfAWS = process.env.AWS_ON_AIR ? it.skip : it; describe('unsupported checksum requests:', () => { before(done => { - makeS3Request({ - method: 'PUT', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'PUT', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); after(done => { - makeS3Request({ - method: 'DELETE', - authCredentials, - bucket, - }, err => { - assert.ifError(err); - done(); - }); + makeS3Request( + { + method: 'DELETE', + authCredentials, + bucket, + }, + err => { + assert.ifError(err); + done(); + }, + ); }); itSkipIfAWS('should respond with BadRequest for trailing checksum', done => { @@ -48,13 +54,13 @@ describe('unsupported checksum requests:', () => { 'x-amz-trailer': 'x-amz-checksum-sha256', }, }, - authCredentials + authCredentials, ), res => { assert.strictEqual(res.statusCode, 400); res.on('data', () => {}); res.on('end', done); - } + }, ); req.on('error', err => { diff --git a/tests/functional/raw-node/test/unsupportedQuries.js b/tests/functional/raw-node/test/unsupportedQuries.js index 9f3995f666..20e09f72ce 100644 --- a/tests/functional/raw-node/test/unsupportedQuries.js +++ b/tests/functional/raw-node/test/unsupportedQuries.js @@ -12,10 +12,8 @@ describe('unsupported query requests:', () => { const queryObj = {}; queryObj[query] = ''; - itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, - done => { - makeS3Request({ method: 'GET', queryObj, bucket, objectKey }, - err => { + itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, done => { + makeS3Request({ method: 'GET', queryObj, bucket, objectKey }, err => { assert.strictEqual(err.code, 'NotImplemented'); assert.strictEqual(err.statusCode, 501); done(); @@ -29,10 +27,8 @@ describe('unsupported bucket query requests:', () => { const queryObj = {}; queryObj[query] = ''; - itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, - done => { - makeS3Request({ method: 'GET', queryObj, bucket }, - err => { + itSkipIfAWS(`should respond with NotImplemented for ?${query} request`, done => { + makeS3Request({ method: 'GET', queryObj, bucket }, err => { assert.strictEqual(err.code, 'NotImplemented'); assert.strictEqual(err.statusCode, 501); done(); diff --git a/tests/functional/raw-node/utils/HttpRequestAuthV4.js b/tests/functional/raw-node/utils/HttpRequestAuthV4.js index b979fd8ce2..0b83bad1e3 100644 --- a/tests/functional/raw-node/utils/HttpRequestAuthV4.js +++ b/tests/functional/raw-node/utils/HttpRequestAuthV4.js @@ -63,30 +63,24 @@ class HttpRequestAuthV4 extends stream.Writable { getCredentialScope() { const signingDate = this._timestamp.slice(0, 8); - const credentialScope = - `${signingDate}/${REGION}/${SERVICE}/aws4_request`; + const credentialScope = `${signingDate}/${REGION}/${SERVICE}/aws4_request`; // console.log(`CREDENTIAL SCOPE: "${credentialScope}"`); return credentialScope; } getSigningKey() { const signingDate = this._timestamp.slice(0, 8); - const dateKey = crypto.createHmac('sha256', `AWS4${this._secretKey}`) - .update(signingDate, 'binary').digest(); - const dateRegionKey = crypto.createHmac('sha256', dateKey) - .update(REGION, 'binary').digest(); - const dateRegionServiceKey = crypto.createHmac('sha256', dateRegionKey) - .update(SERVICE, 'binary').digest(); - this._signingKey = crypto.createHmac('sha256', dateRegionServiceKey) - .update('aws4_request', 'binary').digest(); + const dateKey = crypto.createHmac('sha256', `AWS4${this._secretKey}`).update(signingDate, 'binary').digest(); + const dateRegionKey = crypto.createHmac('sha256', dateKey).update(REGION, 'binary').digest(); + const dateRegionServiceKey = crypto.createHmac('sha256', dateRegionKey).update(SERVICE, 'binary').digest(); + this._signingKey = crypto.createHmac('sha256', dateRegionServiceKey).update('aws4_request', 'binary').digest(); } createSignature(stringToSign) { if (!this._signingKey) { this.getSigningKey(); } - return crypto.createHmac('sha256', this._signingKey) - .update(stringToSign).digest('hex'); + return crypto.createHmac('sha256', this._signingKey).update(stringToSign).digest('hex'); } getCanonicalRequest(urlObj, signedHeaders, contentSha256) { @@ -96,19 +90,16 @@ class HttpRequestAuthV4 extends stream.Writable { urlObj.searchParams.forEach((value, key) => { qsParams.push({ key, value }); }); - const canonicalQueryString = - qsParams - .sort((a, b) => { - if (a.key !== b.key) { - return a.key < b.key ? -1 : 1; - } - return a.value < b.value ? -1 : 1; - }) - .map(param => `${encodeURI(param.key)}=${encodeURI(param.value)}`) - .join('&'); - const canonicalSignedHeaders = signedHeadersList - .map(header => `${header}:${signedHeaders[header]}\n`) - .join(''); + const canonicalQueryString = qsParams + .sort((a, b) => { + if (a.key !== b.key) { + return a.key < b.key ? -1 : 1; + } + return a.value < b.value ? -1 : 1; + }) + .map(param => `${encodeURI(param.key)}=${encodeURI(param.value)}`) + .join('&'); + const canonicalSignedHeaders = signedHeadersList.map(header => `${header}:${signedHeaders[header]}\n`).join(''); const canonicalRequest = [ method, urlObj.pathname, @@ -123,41 +114,37 @@ class HttpRequestAuthV4 extends stream.Writable { } constructRequestStringToSign(canonicalReq) { - const canonicalReqHash = - crypto.createHash('sha256').update(canonicalReq).digest('hex'); - const stringToSign = `AWS4-HMAC-SHA256\n${this._timestamp}\n` + - `${this.getCredentialScope()}\n${canonicalReqHash}`; + const canonicalReqHash = crypto.createHash('sha256').update(canonicalReq).digest('hex'); + const stringToSign = + `AWS4-HMAC-SHA256\n${this._timestamp}\n` + `${this.getCredentialScope()}\n${canonicalReqHash}`; // console.log(`STRING TO SIGN: "${stringToSign}"`); return stringToSign; } getAuthorizationSignature(urlObj, signedHeaders, contentSha256) { - const canonicalRequest = - this.getCanonicalRequest(urlObj, signedHeaders, contentSha256); - this._lastSignature = this.createSignature( - this.constructRequestStringToSign(canonicalRequest)); + const canonicalRequest = this.getCanonicalRequest(urlObj, signedHeaders, contentSha256); + this._lastSignature = this.createSignature(this.constructRequestStringToSign(canonicalRequest)); return this._lastSignature; } getAuthorizationHeader(urlObj, signedHeaders, contentSha256) { - const authorizationSignature = - this.getAuthorizationSignature(urlObj, signedHeaders, contentSha256); + const authorizationSignature = this.getAuthorizationSignature(urlObj, signedHeaders, contentSha256); const signedHeadersList = Object.keys(signedHeaders).sort(); - return ['AWS4-HMAC-SHA256', - `Credential=${this._accessKey}/${this.getCredentialScope()},`, - `SignedHeaders=${signedHeadersList.join(';')},`, - `Signature=${authorizationSignature}`, - ].join(' '); + return [ + 'AWS4-HMAC-SHA256', + `Credential=${this._accessKey}/${this.getCredentialScope()},`, + `SignedHeaders=${signedHeadersList.join(';')},`, + `Signature=${authorizationSignature}`, + ].join(' '); } constructChunkStringToSign(chunkData) { - const currentChunkHash = - crypto.createHash('sha256').update(chunkData.toString()) - .digest('hex'); - const stringToSign = `AWS4-HMAC-SHA256-PAYLOAD\n${this._timestamp}\n` + - `${this.getCredentialScope()}\n${this._lastSignature}\n` + - `${EMPTY_STRING_HASH}\n${currentChunkHash}`; + const currentChunkHash = crypto.createHash('sha256').update(chunkData.toString()).digest('hex'); + const stringToSign = + `AWS4-HMAC-SHA256-PAYLOAD\n${this._timestamp}\n` + + `${this.getCredentialScope()}\n${this._lastSignature}\n` + + `${EMPTY_STRING_HASH}\n${currentChunkHash}`; // console.log(`CHUNK STRING TO SIGN: "${stringToSign}"`); return stringToSign; } @@ -173,13 +160,7 @@ class HttpRequestAuthV4 extends stream.Writable { return chunkData; } const chunkSignature = this.getChunkSignature(chunkData); - return [chunkData.length.toString(16), - ';chunk-signature=', - chunkSignature, - '\r\n', - chunkData, - '\r\n', - ].join(''); + return [chunkData.length.toString(16), ';chunk-signature=', chunkSignature, '\r\n', chunkData, '\r\n'].join(''); } _constructRequest(hasDataToSend) { @@ -196,7 +177,7 @@ class HttpRequestAuthV4 extends stream.Writable { const urlObj = new url.URL(this._url); const signedHeaders = { - 'host': urlObj.host, + host: urlObj.host, 'x-amz-date': this._timestamp, }; const httpHeaders = Object.assign({}, this._httpParams.headers); @@ -212,13 +193,11 @@ class HttpRequestAuthV4 extends stream.Writable { }); if (!signedHeaders['x-amz-content-sha256']) { if (hasDataToSend) { - signedHeaders['x-amz-content-sha256'] = - 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; + signedHeaders['x-amz-content-sha256'] = 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD'; signedHeaders['content-encoding'] = 'aws-chunked'; this._chunkedUpload = true; if (contentLengthHeader !== undefined) { - signedHeaders['x-amz-decoded-content-length'] = - httpHeaders[contentLengthHeader]; + signedHeaders['x-amz-decoded-content-length'] = httpHeaders[contentLengthHeader]; delete signedHeaders['content-length']; delete httpHeaders[contentLengthHeader]; httpHeaders['transfer-encoding'] = 'chunked'; @@ -227,8 +206,11 @@ class HttpRequestAuthV4 extends stream.Writable { signedHeaders['x-amz-content-sha256'] = EMPTY_STRING_HASH; } } - httpHeaders.Authorization = - this.getAuthorizationHeader(urlObj, signedHeaders, signedHeaders['x-amz-content-sha256']); + httpHeaders.Authorization = this.getAuthorizationHeader( + urlObj, + signedHeaders, + signedHeaders['x-amz-content-sha256'], + ); return Object.assign(httpHeaders, signedHeaders); } diff --git a/tests/functional/raw-node/utils/MetadataMock.js b/tests/functional/raw-node/utils/MetadataMock.js index 008a284b41..6b0a661b80 100644 --- a/tests/functional/raw-node/utils/MetadataMock.js +++ b/tests/functional/raw-node/utils/MetadataMock.js @@ -8,10 +8,10 @@ const dummyBucketMD = { WRITE: [], WRITE_ACP: [], READ: [], - READ_ACP: [] }, + READ_ACP: [], + }, _name: 'xxxfriday10', - _owner: - '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', + _owner: '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', _ownerDisplayName: 'test_1518720219', _creationDate: '2018-02-16T21:55:16.415Z', _mdBucketModelVersion: 5, @@ -33,10 +33,10 @@ const dummyBucketMD = { WRITE: [], WRITE_ACP: [], READ: [], - READ_ACP: [] }, + READ_ACP: [], + }, _name: 'xxxfriday11', - _owner: - '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', + _owner: '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', _ownerDisplayName: 'test_1518720219', _creationDate: '2018-02-16T21:55:16.415Z', _mdBucketModelVersion: 5, @@ -55,157 +55,207 @@ const dummyBucketMD = { const objectList = { Contents: [ - { key: 'testobject1', - value: JSON.stringify({ - 'owner-display-name': 'test_1518720219', - 'owner-id': - '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', - 'content-length': 0, - 'content-md5': 'd41d8cd98f00b204e9800998ecf8427e', - 'x-amz-version-id': 'null', - 'x-amz-server-version-id': '', - 'x-amz-storage-class': 'STANDARD', - 'x-amz-server-side-encryption': '', - 'x-amz-server-side-encryption-aws-kms-key-id': '', - 'x-amz-server-side-encryption-customer-algorithm': '', - 'x-amz-website-redirect-location': '', - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - 'key': '', - 'location': null, - 'isDeleteMarker': false, - 'tags': {}, - 'replicationInfo': { - status: '', - backends: [], - content: [], - destination: '', - storageClass: '', - role: '', - storageType: '', - dataStoreVersionId: '', - }, - 'dataStoreName': 'us-east-1', - 'last-modified': '2018-02-16T22:43:37.174Z', - 'md-model-version': 3, - }) }, + { + key: 'testobject1', + value: JSON.stringify({ + 'owner-display-name': 'test_1518720219', + 'owner-id': '94224c921648ada653f584f3caf42654ccf3f1cbd2e569a24e88eb460f2f84d8', + 'content-length': 0, + 'content-md5': 'd41d8cd98f00b204e9800998ecf8427e', + 'x-amz-version-id': 'null', + 'x-amz-server-version-id': '', + 'x-amz-storage-class': 'STANDARD', + 'x-amz-server-side-encryption': '', + 'x-amz-server-side-encryption-aws-kms-key-id': '', + 'x-amz-server-side-encryption-customer-algorithm': '', + 'x-amz-website-redirect-location': '', + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + key: '', + location: null, + isDeleteMarker: false, + tags: {}, + replicationInfo: { + status: '', + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, + dataStoreName: 'us-east-1', + 'last-modified': '2018-02-16T22:43:37.174Z', + 'md-model-version': 3, + }), + }, ], }; const mockLogs = { info: { start: 1, cseq: 7, prune: 1 }, log: [ - { db: 'friday', method: 0, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":true,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\"}' }, - ] }, - { db: 'friday', method: 7, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":false,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\",' + - '\"raftSession\":1}' }, - ] }, - { db: 'friday7', method: 0, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":true,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\"}' }, - ] }, - { db: 'friday7', method: 7, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":false,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\",' + - '\"raftSession\":1}' }, - ] }, - { db: 'xxxfriday10', method: 0, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":true,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\"}' }, - ] }, - { db: 'xxxfriday10', method: 7, entries: [ - { value: '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + - '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + - '\\\"test_1518720219\\\",\\\"creationDate\\\":' + - '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + - '\\\"transient\\\":false,\\\"deleted\\\":false,' + - '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + - '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + - '\\\":null,\\\"replicationConfiguration\\\":null}\",' + - '\"raftSession\":1}' }, - ] }, - { db: 'xxxfriday10', method: 8, entries: [ - { - key: 'afternoon', - value: '{\"owner-display-name\":\"test_1518720219\",' + - '\"owner-id\":\"94224c921648ada653f584f3caf42654ccf3f1cb' + - 'd2e569a24e88eb460f2f84d8\",\"content-length\":0,' + - '\"content-md5\":\"d41d8cd98f00b204e9800998ecf8427e\",' + - '\"x-amz-version-id\":\"null\",' + - '\"x-amz-server-version-id\":\"\",\"x-amz-storage-class' + - '\":\"STANDARD\",\"x-amz-server-side-encryption\":\"\",' + - '\"x-amz-server-side-encryption-aws-kms-key-id\":\"\",' + - '\"x-amz-server-side-encryption-customer-algorithm\":' + - '\"\",\"x-amz-website-redirect-location\":\"\",\"acl\":' + - '{\"Canned\":\"private\",\"FULL_CONTROL\":[],' + - '\"WRITE_ACP\":[],\"READ\":[],\"READ_ACP\":[]},\"key\":' + - '\"\",\"location\":null,\"isDeleteMarker\":false,\"tags' + - '\":{},\"replicationInfo\":{\"status\":\"\",\"backends\":' + - '[],\"content\":[],\"destination\":\"\",\"storageClass\":' + - '\"\",\"role\":\"\",\"storageType\":\"\",' + - '\"dataStoreVersionId\":\"\"},\"dataStoreName\":' + - '\"us-east-1\",\"last-modified\":\"2018-02-16T21:56:52.' + - '690Z\",\"md-model-version\":3}', - }, - ] }, - ] }; + { + db: 'friday', + method: 0, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":true,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\"}', + }, + ], + }, + { + db: 'friday', + method: 7, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T19:59:31.664Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":false,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\",' + + '\"raftSession\":1}', + }, + ], + }, + { + db: 'friday7', + method: 0, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":true,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\"}', + }, + ], + }, + { + db: 'friday7', + method: 7, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"friday7\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T20:41:34.253Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":false,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\",' + + '\"raftSession\":1}', + }, + ], + }, + { + db: 'xxxfriday10', + method: 0, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":true,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\"}', + }, + ], + }, + { + db: 'xxxfriday10', + method: 7, + entries: [ + { + value: + '{\"attributes\":\"{\\\"name\\\":\\\"xxxfriday10\\\",' + + '\\\"owner\\\":\\\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\\\",\\\"ownerDisplayName\\\":' + + '\\\"test_1518720219\\\",\\\"creationDate\\\":' + + '\\\"2018-02-16T21:55:16.415Z\\\",\\\"mdBucketModelVersion\\\":5,' + + '\\\"transient\\\":false,\\\"deleted\\\":false,' + + '\\\"serverSideEncryption\\\":null,\\\"versioningConfiguration' + + '\\\":null,\\\"locationConstraint\\\":\\\"us-east-1\\\",\\\"cors' + + '\\\":null,\\\"replicationConfiguration\\\":null}\",' + + '\"raftSession\":1}', + }, + ], + }, + { + db: 'xxxfriday10', + method: 8, + entries: [ + { + key: 'afternoon', + value: + '{\"owner-display-name\":\"test_1518720219\",' + + '\"owner-id\":\"94224c921648ada653f584f3caf42654ccf3f1cb' + + 'd2e569a24e88eb460f2f84d8\",\"content-length\":0,' + + '\"content-md5\":\"d41d8cd98f00b204e9800998ecf8427e\",' + + '\"x-amz-version-id\":\"null\",' + + '\"x-amz-server-version-id\":\"\",\"x-amz-storage-class' + + '\":\"STANDARD\",\"x-amz-server-side-encryption\":\"\",' + + '\"x-amz-server-side-encryption-aws-kms-key-id\":\"\",' + + '\"x-amz-server-side-encryption-customer-algorithm\":' + + '\"\",\"x-amz-website-redirect-location\":\"\",\"acl\":' + + '{\"Canned\":\"private\",\"FULL_CONTROL\":[],' + + '\"WRITE_ACP\":[],\"READ\":[],\"READ_ACP\":[]},\"key\":' + + '\"\",\"location\":null,\"isDeleteMarker\":false,\"tags' + + '\":{},\"replicationInfo\":{\"status\":\"\",\"backends\":' + + '[],\"content\":[],\"destination\":\"\",\"storageClass\":' + + '\"\",\"role\":\"\",\"storageType\":\"\",' + + '\"dataStoreVersionId\":\"\"},\"dataStoreName\":' + + '\"us-east-1\",\"last-modified\":\"2018-02-16T21:56:52.' + + '690Z\",\"md-model-version\":3}', + }, + ], + }, + ], +}; -const mockLogString = '\\/_\\/raft_sessions\\/[\\d]*\\/log\\?begin=' + - '[\\d]*&limit=[\\d]*&targetLeader=false'; +const mockLogString = '\\/_\\/raft_sessions\\/[\\d]*\\/log\\?begin=' + '[\\d]*&limit=[\\d]*&targetLeader=false'; const mockLogURLRegex = new RegExp(mockLogString); class MetadataMock { onRequest(req, res) { if (req.method !== 'GET') { res.writeHead(501); - return res.end(JSON.stringify({ - error: 'mock server only supports GET requests', - })); + return res.end( + JSON.stringify({ + error: 'mock server only supports GET requests', + }), + ); } if (/\/_\/raft_sessions\/[1-8]\/bucket/.test(req.url)) { const value = ['bucket1', 'bucket2', 'users..bucket']; @@ -214,30 +264,41 @@ class MetadataMock { } else if (/\/default\/attributes\/[a-z0-9]/.test(req.url)) { const bucketName = req.url.split('/'); const bucketMd = dummyBucketMD[bucketName[bucketName.length - 1]]; - const dummyBucketMdObj = new BucketInfo(bucketMd._name, - bucketMd._owner, bucketMd._ownerDisplayName, - bucketMd._creationDate, bucketMd._mdBucketModelVersion, - bucketMd._acl, bucketMd._transient, bucketMd._deleted, + const dummyBucketMdObj = new BucketInfo( + bucketMd._name, + bucketMd._owner, + bucketMd._ownerDisplayName, + bucketMd._creationDate, + bucketMd._mdBucketModelVersion, + bucketMd._acl, + bucketMd._transient, + bucketMd._deleted, bucketMd._serverSideEncryption, - bucketMd.versioningConfiguration, bucketMd._locationContraint, - bucketMd._websiteConfiguration, bucketMd._cors, - bucketMd._lifeCycle); + bucketMd.versioningConfiguration, + bucketMd._locationContraint, + bucketMd._websiteConfiguration, + bucketMd._cors, + bucketMd._lifeCycle, + ); return res.end(dummyBucketMdObj.serialize()); - } else if - (/\/default\/bucket\/.*?listingType=Delimiter/.test(req.url)) { + } else if (/\/default\/bucket\/.*?listingType=Delimiter/.test(req.url)) { return res.end(JSON.stringify(objectList)); } else if (/\/default\/bucket\/.*\/.*?/.test(req.url)) { - return res.end(JSON.stringify({ - 'owner-id': '123', - 'metadata': 'dogsAreGood', - })); + return res.end( + JSON.stringify({ + 'owner-id': '123', + metadata: 'dogsAreGood', + }), + ); } else if (mockLogURLRegex.test(req.url)) { return res.end(JSON.stringify(mockLogs)); } res.writeHead(404); - return res.end(JSON.stringify({ - error: 'invalid path', - })); + return res.end( + JSON.stringify({ + error: 'invalid path', + }), + ); } } diff --git a/tests/functional/raw-node/utils/gcpUtils.js b/tests/functional/raw-node/utils/gcpUtils.js index 773fd0ec83..ea6e1d40a7 100644 --- a/tests/functional/raw-node/utils/gcpUtils.js +++ b/tests/functional/raw-node/utils/gcpUtils.js @@ -11,8 +11,7 @@ const genUniqID = () => { const genBucketName = testName => `cldsrvci-${testName}-${genUniqID()}`; -const defaultShouldRetry = err => - err && (err.name === 'SlowDown' || err.$metadata?.httpStatusCode === 429); +const defaultShouldRetry = err => err && (err.name === 'SlowDown' || err.$metadata?.httpStatusCode === 429); async function gcpRetryCall(callFn, retryOptions) { const { @@ -33,9 +32,9 @@ async function gcpRetryCall(callFn, retryOptions) { } const delay = getDelayMs(attempt); process.stdout.write( - 'Retryable error from GCP, retrying in ' + - `${delay}ms (attempt ${attempt + 1}): ${err}\n`); - + 'Retryable error from GCP, retrying in ' + `${delay}ms (attempt ${attempt + 1}): ${err}\n`, + ); + await new Promise(resolve => setTimeout(resolve, delay)); } } @@ -45,41 +44,40 @@ async function gcpRetryCall(callFn, retryOptions) { async function gcpRetry(gcpClient, command, retryOptions, cb) { if (cb) { - return callbackify(() => gcpRetry(gcpClient, command, - retryOptions))(cb); + return callbackify(() => gcpRetry(gcpClient, command, retryOptions))(cb); } return gcpRetryCall(() => gcpClient.send(command), retryOptions); } -const defaultShouldRetryUpload = err => err && ( - err.name === 'NoSuchBucket' - || err.name === 'NotFound' - || err.$metadata?.httpStatusCode === 404 - || err.name === 'SlowDown' - || err.$metadata?.httpStatusCode === 429 - || (typeof err.message === 'string' - && (err.message.includes('NoSuchBucket') - || err.message.includes('unable to complete upload'))) -); - -const defaultShouldRetryMpuCreate = err => err && ( - err.name === 'NoSuchBucket' - || err.name === 'NotFound' - || err.$metadata?.httpStatusCode === 404 - || err.name === 'SlowDown' - || err.$metadata?.httpStatusCode === 429 -); +const defaultShouldRetryUpload = err => + err && + (err.name === 'NoSuchBucket' || + err.name === 'NotFound' || + err.$metadata?.httpStatusCode === 404 || + err.name === 'SlowDown' || + err.$metadata?.httpStatusCode === 429 || + (typeof err.message === 'string' && + (err.message.includes('NoSuchBucket') || err.message.includes('unable to complete upload')))); + +const defaultShouldRetryMpuCreate = err => + err && + (err.name === 'NoSuchBucket' || + err.name === 'NotFound' || + err.$metadata?.httpStatusCode === 404 || + err.name === 'SlowDown' || + err.$metadata?.httpStatusCode === 429); async function gcpUploadWithRetry(gcpClient, params, retryOptions) { - const callFn = () => new Promise((resolve, reject) => { - gcpClient.upload(params, (err, data) => { - if (err) { - return reject(err); - } - return resolve(data); + const callFn = () => + new Promise((resolve, reject) => { + gcpClient.upload(params, (err, data) => { + if (err) { + return reject(err); + } + return resolve(data); + }); }); - }); return gcpRetryCall(callFn, { maxAttempts: 6, @@ -90,10 +88,10 @@ async function gcpUploadWithRetry(gcpClient, params, retryOptions) { } async function gcpCreateMultipartUploadWithRetry(gcpClient, params, retryOptions) { - const callFn = () => new Promise((resolve, reject) => { - gcpClient.createMultipartUpload(params, - (err, res) => (err ? reject(err) : resolve(res))); - }); + const callFn = () => + new Promise((resolve, reject) => { + gcpClient.createMultipartUpload(params, (err, res) => (err ? reject(err) : resolve(res))); + }); return gcpRetryCall(callFn, { maxAttempts: 6, shouldRetry: defaultShouldRetryMpuCreate, @@ -106,52 +104,63 @@ async function gcpCreateMultipartUploadWithRetry(gcpClient, params, retryOptions function gcpMpuSetup(params, callback) { const { gcpClient, bucketNames, key, partCount, partSize } = params; - return async.waterfall([ - next => gcpCreateMultipartUploadWithRetry(gcpClient, { - Bucket: bucketNames.mpu.Name, - Key: key, - }) - .then(res => next(null, res.UploadId)) - .catch(err => next(err)), - (uploadId, next) => { - if (partCount <= 0) { - return next('SkipPutPart', { uploadId }); - } - const arrayData = Array.from(Array(partCount).keys()); - const etagList = Array(partCount); - let count = 0; - return async.eachLimit(arrayData, 10, - (info, moveOn) => { - gcpClient.uploadPart({ + return async.waterfall( + [ + next => + gcpCreateMultipartUploadWithRetry(gcpClient, { Bucket: bucketNames.mpu.Name, Key: key, - UploadId: uploadId, - PartNumber: info + 1, - Body: Buffer.alloc(partSize), - ContentLength: partSize, - }, (err, res) => { - if (err) { - return moveOn(err); - } - if (!(++count % 100)) { - process.stdout.write(`Uploaded Parts: ${count}\n`); - } - etagList[info] = res.ETag; - return moveOn(null); - }); - }, err => { - next(err, { uploadId, etagList }); - }); - }, - ], (err, result) => { - if (err) { - if (err === 'SkipPutPart') { - return callback(null, result); + }) + .then(res => next(null, res.UploadId)) + .catch(err => next(err)), + (uploadId, next) => { + if (partCount <= 0) { + return next('SkipPutPart', { uploadId }); + } + const arrayData = Array.from(Array(partCount).keys()); + const etagList = Array(partCount); + let count = 0; + return async.eachLimit( + arrayData, + 10, + (info, moveOn) => { + gcpClient.uploadPart( + { + Bucket: bucketNames.mpu.Name, + Key: key, + UploadId: uploadId, + PartNumber: info + 1, + Body: Buffer.alloc(partSize), + ContentLength: partSize, + }, + (err, res) => { + if (err) { + return moveOn(err); + } + if (!(++count % 100)) { + process.stdout.write(`Uploaded Parts: ${count}\n`); + } + etagList[info] = res.ETag; + return moveOn(null); + }, + ); + }, + err => { + next(err, { uploadId, etagList }); + }, + ); + }, + ], + (err, result) => { + if (err) { + if (err === 'SkipPutPart') { + return callback(null, result); + } + return callback(err); } - return callback(err); - } - return callback(null, result); - }); + return callback(null, result); + }, + ); } function genPutTagObj(size, duplicate) { @@ -200,12 +209,13 @@ function genDelTagObj(size, tagPrefix) { const regionalLoc = 'us-west1'; const multiRegionalLoc = 'us'; function setBucketClass(storageClass) { - const locationConstraint = - storageClass === 'REGIONAL' ? regionalLoc : multiRegionalLoc; - return '' + + const locationConstraint = storageClass === 'REGIONAL' ? regionalLoc : multiRegionalLoc; + return ( + '' + `${locationConstraint}` + `${storageClass}` + - ''; + '' + ); } async function waitForBucketReady(gcpClient, bucketName, retryOptions) { diff --git a/tests/functional/report/master.json b/tests/functional/report/master.json index cc83b902dd..9fe2fc66b9 100644 --- a/tests/functional/report/master.json +++ b/tests/functional/report/master.json @@ -1,6 +1,6 @@ { "tests": { - "files": [ "/test" ], + "files": ["/test"], "on": "aggressor" } } diff --git a/tests/functional/report/monitoring.js b/tests/functional/report/monitoring.js index 529aeaebef..4a5bcf4ca1 100644 --- a/tests/functional/report/monitoring.js +++ b/tests/functional/report/monitoring.js @@ -7,13 +7,20 @@ describe('Monitoring - getting metrics', () => { const conf = require('../config.json'); async function query(path, method = 'GET', token = 'report-token-1') { - return new Promise(resolve => http.request({ - method, - host: conf.ipAddress, - path, - port: 8000, - headers: { 'x-scal-report-token': token }, - }, () => resolve()).end()); + return new Promise(resolve => + http + .request( + { + method, + host: conf.ipAddress, + path, + port: 8000, + headers: { 'x-scal-report-token': token }, + }, + () => resolve(), + ) + .end(), + ); } async function getMetrics() { @@ -22,14 +29,18 @@ describe('Monitoring - getting metrics', () => { assert.strictEqual(res.statusCode, 200); const body = []; - res.on('data', chunk => { body.push(chunk); }); + res.on('data', chunk => { + body.push(chunk); + }); res.on('end', () => resolve(body.join(''))); }); }); } function parseMetric(metrics, name, labels) { - const labelsString = Object.entries(labels).map(e => `${e[0]}="${e[1]}"`).join(','); + const labelsString = Object.entries(labels) + .map(e => `${e[0]}="${e[1]}"`) + .join(','); const metric = metrics.match(new RegExp(`^${name}{${labelsString}} (.*)$`, 'm')); return metric ? metric[1] : null; } @@ -51,26 +62,26 @@ describe('Monitoring - getting metrics', () => { [ // Check all methods are reported (on unsupported route) - ['/_/fooooo', { method: 'GET', code: '400' }], - ['/_/fooooo', { method: 'PUT', code: '400' }], - ['/_/fooooo', { method: 'POST', code: '400' }], - ['/_/fooooo', { method: 'DELETE', code: '400' }], + ['/_/fooooo', { method: 'GET', code: '400' }], + ['/_/fooooo', { method: 'PUT', code: '400' }], + ['/_/fooooo', { method: 'POST', code: '400' }], + ['/_/fooooo', { method: 'DELETE', code: '400' }], // S3/api routes - ['/', { method: 'GET', code: '403', action: 'serviceGet' }], - ['/foo', { method: 'GET', code: '404', action: 'bucketGet' }], - ['/foo/bar', { method: 'GET', code: '404', action: 'objectGet' }], + ['/', { method: 'GET', code: '403', action: 'serviceGet' }], + ['/foo', { method: 'GET', code: '404', action: 'bucketGet' }], + ['/foo/bar', { method: 'GET', code: '404', action: 'objectGet' }], // Internal handlers - ['/_/report', { method: 'GET', code: '200', action: 'report' }], - ['/_/backbeat', { method: 'GET', code: '405', action: 'routeBackbeat' }], - ['/_/metadata', { method: 'GET', code: '403', action: 'routeMetadata' }], - ['/_/workflow-engine-operator', - { method: 'GET', code: '405', action: 'routeWorkflowEngineOperator' }], + ['/_/report', { method: 'GET', code: '200', action: 'report' }], + ['/_/backbeat', { method: 'GET', code: '405', action: 'routeBackbeat' }], + ['/_/metadata', { method: 'GET', code: '403', action: 'routeMetadata' }], + ['/_/workflow-engine-operator', { method: 'GET', code: '405', action: 'routeWorkflowEngineOperator' }], ].forEach(([path, labels]) => { it(`should count http ${labels.method} requests metrics on ${path}`, async () => { const count = parseRequestsCount(await getMetrics(), labels); - for (let i = 1; i <= 3; i++) { /* eslint no-await-in-loop: "off" */ + for (let i = 1; i <= 3; i++) { + /* eslint no-await-in-loop: "off" */ await query(path, labels.method); const c = parseRequestsCount(await getMetrics(), labels); diff --git a/tests/functional/s3cmd/tests.js b/tests/functional/s3cmd/tests.js index 9140714ad6..9d05cfceb3 100644 --- a/tests/functional/s3cmd/tests.js +++ b/tests/functional/s3cmd/tests.js @@ -14,11 +14,7 @@ const emptyUpload = 'Utest0B'; const emptyDownload = 'Dtest0B'; const download = 'tmpfile'; const MPUpload = 'test60MB'; -const MPUploadSplitter = [ - 'test60..|..MB', - '..|..test60MB', - 'test60MB..|..', -]; +const MPUploadSplitter = ['test60..|..MB', '..|..test60MB', 'test60MB..|..']; const MPDownload = 'MPtmpfile'; const MPDownloadCopy = 'MPtmpfile2'; const downloadCopy = 'tmpfile2'; @@ -51,8 +47,7 @@ function diff(putFile, receivedFile, done) { function createFile(name, bytes, callback) { process.stdout.write(`dd if=/dev/urandom of=${name} bs=${bytes} count=1\n`); - const ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, - `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); + const ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); assert.strictEqual(ret.status, 0); callback(); } @@ -81,8 +76,7 @@ function exec(args, done, exitCode) { } process.stdout.write(`${program} ${av}\n`); const ret = proc.spawnSync(program, av, { stdio: 'inherit' }); - assert.strictEqual(ret.status, exit, - 's3cmd did not yield expected exit status.'); + assert.strictEqual(ret.status, exit, 's3cmd did not yield expected exit status.'); done(); } @@ -106,12 +100,16 @@ function checkRawOutput(args, lineFinder, testString, stream, cb) { }); child.on('close', () => { if (stream === 'stderr') { - const foundIt = allErrData.join('').split('\n') + const foundIt = allErrData + .join('') + .split('\n') .filter(item => item.indexOf(lineFinder) > -1) .some(item => item.indexOf(testString) > -1); return cb(foundIt); } - const foundIt = allData.join('').split('\n') + const foundIt = allData + .join('') + .split('\n') .filter(item => item.indexOf(lineFinder) > -1) .some(item => item.indexOf(testString) > -1); return cb(foundIt); @@ -123,12 +121,12 @@ function findEndString(data, start) { const end = data.length; for (let i = start + 1; i < end; ++i) { if (data[i] === delimiter) { - return (i); + return i; } else if (data[i] === '\\') { ++i; } } - return (-1); + return -1; } function findEndJson(data, start) { @@ -143,10 +141,10 @@ function findEndJson(data, start) { i = findEndString(data, i); } if (count === 0) { - return (i); + return i; } } - return (-1); + return -1; } function readJsonFromChild(child, lineFinder, cb) { @@ -160,9 +158,11 @@ function readJsonFromChild(child, lineFinder, cb) { const findLine = data.indexOf(lineFinder); const findBrace = data.indexOf('{', findLine); const findEnd = findEndJson(data, findBrace); - const endJson = data.substring(findBrace, findEnd + 1) - .replace(/"/g, '\\"').replace(/'/g, '"') - .replace(/b'/g, '\'') + const endJson = data + .substring(findBrace, findEnd + 1) + .replace(/"/g, '\\"') + .replace(/'/g, '"') + .replace(/b'/g, "'") .replace(/b"/g, '"'); return cb(JSON.parse(endJson)); }); @@ -213,37 +213,32 @@ function retrieveInfo() { function createEncryptedBucket(name, cb) { const res = retrieveInfo(); const prog = `${__dirname}/../../../bin/create_encrypted_bucket.js`; - let args = [ - prog, - '-a', res.accessKey, - '-k', res.secretKey, - '-b', name, - '-h', res.host, - '-p', res.port, - '-v', - ]; + let args = [prog, '-a', res.accessKey, '-k', res.secretKey, '-b', name, '-h', res.host, '-p', res.port, '-v']; if (conf.https) { args = args.concat('-s'); } const body = []; - const child = proc.spawn(args[0], args) - .on('exit', () => { - const hasSucceed = body.join('').split('\n').find(item => { - const json = safeJSONParse(item); - const test = !(json instanceof Error) && json.name === 'S3' && - json.statusCode === 200; - if (test) { - return true; + const child = proc + .spawn(args[0], args) + .on('exit', () => { + const hasSucceed = body + .join('') + .split('\n') + .find(item => { + const json = safeJSONParse(item); + const test = !(json instanceof Error) && json.name === 'S3' && json.statusCode === 200; + if (test) { + return true; + } + return false; + }); + if (!hasSucceed) { + process.stderr.write(`${body.join('')}\n`); + return cb(new Error('Cannot create encrypted bucket')); } - return false; - }); - if (!hasSucceed) { - process.stderr.write(`${body.join('')}\n`); - return cb(new Error('Cannot create encrypted bucket')); - } - return cb(); - }) - .on('error', cb); + return cb(); + }) + .on('error', cb); child.stdout.on('data', chunk => body.push(chunk.toString())); } @@ -257,10 +252,7 @@ describe('s3cmd putBucket', () => { // pass by returning error. If legacyAWSBehvior, request // would return a 200 it('put the same bucket, should fail', done => { - exec([ - 'mb', `s3://${bucket}`, - '--bucket-location=scality-us-west-1', - ], done, 13); + exec(['mb', `s3://${bucket}`, '--bucket-location=scality-us-west-1'], done, 13); }); it('put an invalid bucket, should fail', done => { @@ -276,16 +268,15 @@ describe('s3cmd putBucket', () => { }); if (process.env.ENABLE_KMS_ENCRYPTION === 'true') { - it('creates a valid bucket with server side encryption', - function f(done) { - this.timeout(5000); - exec(['rb', `s3://${bucket}`], err => { - if (err) { - return done(err); - } - return createEncryptedBucket(bucket, done); - }); - }); + it('creates a valid bucket with server side encryption', function f(done) { + this.timeout(5000); + exec(['rb', `s3://${bucket}`], err => { + if (err) { + return done(err); + } + return createEncryptedBucket(bucket, done); + }); + }); } }); @@ -299,23 +290,18 @@ describe('s3cmd put and get bucket ACLs', function aclBuck() { }); it('should get canned ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}`], 'ACL', '*anon*: READ', - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'ACL', '*anon*: READ', 'stdout', foundIt => { assert(foundIt); done(); }); }); it('should set a specific ACL', done => { - exec([ - 'setacl', `s3://${bucket}`, - `--acl-grant=write:${emailAccount}`, - ], done); + exec(['setacl', `s3://${bucket}`, `--acl-grant=write:${emailAccount}`], done); }); it('should get specific ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}`], 'ACL', - `${lowerCaseEmail}: WRITE`, 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'ACL', `${lowerCaseEmail}: WRITE`, 'stdout', foundIt => { assert(foundIt); done(); }); @@ -340,23 +326,18 @@ describe('s3cmd getService', () => { }); }); - it("should have response headers matching AWS's response headers", - done => { - provideLineOfInterest(['ls', '--debug'], '\'headers\': {', - parsedObject => { - assert(parsedObject['x-amz-id-2']); - assert(parsedObject['transfer-encoding']); - assert(parsedObject['x-amz-request-id']); - const gmtDate = new Date(parsedObject.date) - .toUTCString(); - assert.strictEqual(parsedObject.date, gmtDate); - assert.strictEqual(parsedObject - ['content-type'], 'application/xml'); - assert.strictEqual(parsedObject - ['set-cookie'], undefined); - done(); - }); + it("should have response headers matching AWS's response headers", done => { + provideLineOfInterest(['ls', '--debug'], "'headers': {", parsedObject => { + assert(parsedObject['x-amz-id-2']); + assert(parsedObject['transfer-encoding']); + assert(parsedObject['x-amz-request-id']); + const gmtDate = new Date(parsedObject.date).toUTCString(); + assert.strictEqual(parsedObject.date, gmtDate); + assert.strictEqual(parsedObject['content-type'], 'application/xml'); + assert.strictEqual(parsedObject['set-cookie'], undefined); + done(); }); + }); }); describe('s3cmd putObject', function toto() { @@ -409,10 +390,7 @@ describe('s3cmd copyObject without MPU to same bucket', function copyStuff() { }); it('should copy an object to the same bucket', done => { - exec([ - 'cp', `s3://${bucket}/${upload}`, - `s3://${bucket}/${upload}copy`, - ], done); + exec(['cp', `s3://${bucket}/${upload}`, `s3://${bucket}/${upload}copy`], done); }); it('should get an object that was copied', done => { @@ -428,42 +406,36 @@ describe('s3cmd copyObject without MPU to same bucket', function copyStuff() { }); }); -describe('s3cmd copyObject without MPU to different bucket ' + - '(always unencrypted)', - function copyStuff() { - const copyBucket = 'receiverbucket'; - this.timeout(40000); +describe('s3cmd copyObject without MPU to different bucket ' + '(always unencrypted)', function copyStuff() { + const copyBucket = 'receiverbucket'; + this.timeout(40000); - before('create receiver bucket', done => { - exec(['mb', `s3://${copyBucket}`], done); - }); + before('create receiver bucket', done => { + exec(['mb', `s3://${copyBucket}`], done); + }); - after('delete downloaded file and receiver bucket' + - 'copied', done => { - deleteFile(downloadCopy, () => { - exec(['rb', `s3://${copyBucket}`], done); - }); + after('delete downloaded file and receiver bucket' + 'copied', done => { + deleteFile(downloadCopy, () => { + exec(['rb', `s3://${copyBucket}`], done); }); + }); - it('should copy an object to the new bucket', done => { - exec([ - 'cp', `s3://${bucket}/${upload}`, - `s3://${copyBucket}/${upload}`, - ], done); - }); + it('should copy an object to the new bucket', done => { + exec(['cp', `s3://${bucket}/${upload}`, `s3://${copyBucket}/${upload}`], done); + }); - it('should get an object that was copied', done => { - exec(['get', `s3://${copyBucket}/${upload}`, downloadCopy], done); - }); + it('should get an object that was copied', done => { + exec(['get', `s3://${copyBucket}/${upload}`, downloadCopy], done); + }); - it('downloaded copy file should equal original uploaded file', done => { - diff(upload, downloadCopy, done); - }); + it('downloaded copy file should equal original uploaded file', done => { + diff(upload, downloadCopy, done); + }); - it('should delete copy of object', done => { - exec(['rm', `s3://${copyBucket}/${upload}`], done); - }); + it('should delete copy of object', done => { + exec(['rm', `s3://${copyBucket}/${upload}`], done); }); +}); describe('s3cmd put and get object ACLs', function aclObj() { this.timeout(60000); @@ -475,30 +447,25 @@ describe('s3cmd put and get object ACLs', function aclObj() { }); it('should get canned ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', - '*anon*: READ', 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', '*anon*: READ', 'stdout', foundIt => { assert(foundIt); done(); }); }); it('should set a specific ACL', done => { - exec(['setacl', `s3://${bucket}/${upload}`, - `--acl-grant=read:${emailAccount}`], done); + exec(['setacl', `s3://${bucket}/${upload}`, `--acl-grant=read:${emailAccount}`], done); }); it('should get specific ACL that was set', done => { - checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', - `${lowerCaseEmail}: READ`, 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}/${upload}`], 'ACL', `${lowerCaseEmail}: READ`, 'stdout', foundIt => { assert(foundIt); done(); }); }); - it('should return error if set acl for ' + - 'nonexistent object', done => { - exec(['setacl', `s3://${bucket}/${nonexist}`, - '--acl-public'], done, 12); + it('should return error if set acl for ' + 'nonexistent object', done => { + exec(['setacl', `s3://${bucket}/${nonexist}`, '--acl-public'], done, 12); }); }); @@ -508,16 +475,14 @@ describe('s3cmd delObject', () => { }); it('delete an already deleted object, should return a 204', done => { - provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'], - 'DEBUG: Response:\n{', parsedObject => { + provideLineOfInterest(['rm', `s3://${bucket}/${upload}`, '--debug'], 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); }); it('delete non-existing object, should return a 204', done => { - provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'], - 'DEBUG: Response:\n{', parsedObject => { + provideLineOfInterest(['rm', `s3://${bucket}/${nonexist}`, '--debug'], 'DEBUG: Response:\n{', parsedObject => { assert.strictEqual(parsedObject.status, 204); done(); }); @@ -596,10 +561,7 @@ describe('s3cmd multipart upload', function titi() { }); it('should copy an object that was put via multipart upload', done => { - exec([ - 'cp', `s3://${bucket}/${MPUpload}`, - `s3://${bucket}/${MPUpload}copy`, - ], done); + exec(['cp', `s3://${bucket}/${MPUpload}`, `s3://${bucket}/${MPUpload}copy`], done); }); it('should get an object that was copied', done => { @@ -663,9 +625,7 @@ MPUploadSplitter.forEach(file => { }); }); - -describe('s3cmd put, get and delete object with spaces ' + - 'in object key names', function test() { +describe('s3cmd put, get and delete object with spaces ' + 'in object key names', function test() { this.timeout(0); const keyWithSpacesAndPluses = 'key with spaces and + pluses +'; before('create file to put', done => { @@ -688,13 +648,11 @@ describe('s3cmd put, get and delete object with spaces ' + }); it('should get file with spaces', done => { - exec(['get', `s3://${bucket}/${keyWithSpacesAndPluses}`, download], - done); + exec(['get', `s3://${bucket}/${keyWithSpacesAndPluses}`, download], done); }); it('should list bucket showing file with spaces', done => { - checkRawOutput(['ls', `s3://${bucket}`], `s3://${bucket}`, - keyWithSpacesAndPluses, 'stdout', foundIt => { + checkRawOutput(['ls', `s3://${bucket}`], `s3://${bucket}`, keyWithSpacesAndPluses, 'stdout', foundIt => { assert(foundIt); done(); }); @@ -726,27 +684,26 @@ describe('s3cmd info', () => { // test that POLICY and CORS are returned as 'none' it('should find that policy has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none', - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'Policy', 'none', 'stdout', foundIt => { assert(foundIt); done(); }); }); it('should find that cors has a value of none', done => { - checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none', - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'CORS', 'none', 'stdout', foundIt => { assert(foundIt); done(); }); }); describe('after putting cors configuration', () => { - const corsConfig = '' + - 'PUT' + - 'http://www.allowedorigin.com' + - ''; + const corsConfig = + '' + + 'PUT' + + 'http://www.allowedorigin.com' + + ''; const filename = 'corss3cmdfile'; beforeEach(done => { @@ -760,8 +717,7 @@ describe('s3cmd info', () => { }); it('should find that cors has a value', done => { - checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig, - 'stdout', foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'CORS', corsConfig, 'stdout', foundIt => { assert(foundIt, 'Did not find value for cors'); done(); }); @@ -793,12 +749,14 @@ describe('s3cmd recursive delete with objects put by MPU', () => { this.timeout(120000); exec(['mb', `s3://${bucket}`], () => { createFile(upload16MB, 16777216, () => { - async.timesLimit(50, 1, (n, next) => { - exec([ - 'put', upload16MB, `s3://${bucket}/key${n}`, - '--multipart-chunk-size-mb=5', - ], next); - }, done); + async.timesLimit( + 50, + 1, + (n, next) => { + exec(['put', upload16MB, `s3://${bucket}/key${n}`, '--multipart-chunk-size-mb=5'], next); + }, + done, + ); }); }); }); @@ -822,9 +780,7 @@ describeSkipIfE2E('If no location is sent with the request', () => { // WARNING: change "us-east-1" to another locationConstraint depending // on the restEndpoints (./config.json) it('endpoint should be used to determine the locationConstraint', done => { - checkRawOutput(['info', `s3://${bucket}`], 'Location', 'us-east-1', - 'stdout', - foundIt => { + checkRawOutput(['info', `s3://${bucket}`], 'Location', 'us-east-1', 'stdout', foundIt => { assert(foundIt); done(); }); diff --git a/tests/functional/s3curl/tests.js b/tests/functional/s3curl/tests.js index c6562eff97..3d3b858875 100644 --- a/tests/functional/s3curl/tests.js +++ b/tests/functional/s3curl/tests.js @@ -22,8 +22,7 @@ const aclBucket = 'acluniverse'; const nonexist = 'nonexist'; const prefix = 'topLevel'; const delimiter = '/'; -let ownerCanonicalId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d52' - + '18e7cd47ef2be'; +let ownerCanonicalId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d52' + '18e7cd47ef2be'; const endpoint = `${transport}://${ipAddress}:8000`; // Let's precompute a few paths @@ -51,11 +50,9 @@ function diff(putFile, receivedFile, done) { }); } - function createFile(name, bytes, callback) { process.stdout.write(`dd if=/dev/urandom of=${name} bs=${bytes} count=1\n`); - let ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, - `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); + let ret = proc.spawnSync('dd', ['if=/dev/urandom', `of=${name}`, `bs=${bytes}`, 'count=1'], { stdio: 'inherit' }); assert.strictEqual(ret.status, 0); process.stdout.write(`chmod ugoa+rw ${name}\n`); ret = proc.spawnSync('chmod', ['ugo+rw', name], { stdio: 'inherit' }); @@ -101,15 +98,13 @@ function provideRawOutput(args, cb) { httpCode = lines.find(line => { const trimmed = line.trim().toUpperCase(); // ignore 100 Continue HTTP code - if (trimmed.startsWith('HTTP/1.1 ') && - !trimmed.includes('100 CONTINUE')) { + if (trimmed.startsWith('HTTP/1.1 ') && !trimmed.includes('100 CONTINUE')) { return true; } return false; }); if (httpCode) { - httpCode = httpCode.trim().replace('HTTP/1.1 ', '') - .toUpperCase(); + httpCode = httpCode.trim().replace('HTTP/1.1 ', '').toUpperCase(); } else { process.stdout.write(`${lines.join('\n')}\n`); return cb(new Error("Can't find line in http response code")); @@ -134,15 +129,13 @@ function provideRawOutput(args, cb) { * @return {undefined} */ function putObjects(filepath, objectPaths, cb) { - provideRawOutput( - [`--put=${filepath}`, '--', objectPaths[0], '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - if (objectPaths.length > 1) { - return putObjects(filepath, objectPaths.slice(1), cb); - } - return cb(); - }); + provideRawOutput([`--put=${filepath}`, '--', objectPaths[0], '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + if (objectPaths.length > 1) { + return putObjects(filepath, objectPaths.slice(1), cb); + } + return cb(); + }); } /** @@ -157,15 +150,13 @@ function putObjects(filepath, objectPaths, cb) { * @return {undefined} */ function deleteRemoteItems(items, cb) { - provideRawOutput( - ['--delete', '--', items[0], '-v'], - httpCode => { - assert.strictEqual(httpCode, '204 NO CONTENT'); - if (items.length > 1) { - return deleteRemoteItems(items.slice(1), cb); - } - return cb(); - }); + provideRawOutput(['--delete', '--', items[0], '-v'], httpCode => { + assert.strictEqual(httpCode, '204 NO CONTENT'); + if (items.length > 1) { + return deleteRemoteItems(items.slice(1), cb); + } + return cb(); + }); } describe('s3curl put delete buckets', () => { @@ -175,84 +166,68 @@ describe('s3curl put delete buckets', () => { }); it('should put a valid bucket', done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); - it('should return 409 error in new regions and 200 in us-east-1 ' + - '(legacyAWSBehvior) when try to put a bucket with a name ' + - 'already being used', done => { - provideRawOutput(['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert(httpCode === '200 OK' - || httpCode === '409 CONFLICT'); + it( + 'should return 409 error in new regions and 200 in us-east-1 ' + + '(legacyAWSBehvior) when try to put a bucket with a name ' + + 'already being used', + done => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert(httpCode === '200 OK' || httpCode === '409 CONFLICT'); done(); }); - }); + }, + ); - it('should not be able to put a bucket with invalid xml' + - ' in the post body', done => { - provideRawOutput([ - '--createBucket', - '--', - '--data', - 'malformedxml', - bucketPath, - '-v', - ], (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'MalformedXML', - done); - }); + it('should not be able to put a bucket with invalid xml' + ' in the post body', done => { + provideRawOutput( + ['--createBucket', '--', '--data', 'malformedxml', bucketPath, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'MalformedXML', done); + }, + ); }); - it('should not be able to put a bucket with xml that does' + - ' not conform to s3 docs for locationConstraint', done => { - provideRawOutput([ - '--createBucket', - '--', - '--data', - 'a', - bucketPath, - '-v', - ], (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'MalformedXML', - done); - }); - }); + it( + 'should not be able to put a bucket with xml that does' + ' not conform to s3 docs for locationConstraint', + done => { + provideRawOutput( + ['--createBucket', '--', '--data', 'a', bucketPath, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'MalformedXML', done); + }, + ); + }, + ); it('should not be able to put a bucket with an invalid name', done => { - provideRawOutput( - ['--createBucket', '--', `${endpoint}/2`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidBucketName', done); - }); + provideRawOutput(['--createBucket', '--', `${endpoint}/2`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidBucketName', done); + }); }); it('should not be able to put a bucket with an empty name', done => { - provideRawOutput( - ['--createBucket', '--', `${endpoint}/`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); - done(); - }); + provideRawOutput(['--createBucket', '--', `${endpoint}/`, '-v'], httpCode => { + assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); + done(); + }); }); }); describe('s3curl delete bucket', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); after(done => { @@ -264,316 +239,256 @@ describe('s3curl put delete buckets', () => { }); it('should not be able to get a bucket that was deleted', done => { - provideRawOutput( - ['--', bucketPath, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '404 NOT FOUND'); - assertError(rawOutput.stdout, 'NoSuchBucket', done); - }); + provideRawOutput(['--', bucketPath, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '404 NOT FOUND'); + assertError(rawOutput.stdout, 'NoSuchBucket', done); + }); }); - it('should be able to create a bucket with a name' + - 'of a bucket that has previously been deleted', done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + it('should be able to create a bucket with a name' + 'of a bucket that has previously been deleted', done => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); }); describe('s3curl put and get bucket ACLs', () => { after(done => { - deleteRemoteItems([ - `${endpoint}/${aclBucket}`, - `${endpoint}/${aclBucket}2`, - ], done); + deleteRemoteItems([`${endpoint}/${aclBucket}`, `${endpoint}/${aclBucket}2`], done); }); it('should be able to create a bucket with a canned ACL', done => { - provideRawOutput([ - '--createBucket', - '--', - '-H', - 'x-amz-acl:public-read', - `${endpoint}/${aclBucket}`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); - }); - - it('should be able to get a canned ACL', done => { provideRawOutput( - ['--', `${endpoint}/${aclBucket}?acl`, '-v'], - (httpCode, rawOutput) => { + ['--createBucket', '--', '-H', 'x-amz-acl:public-read', `${endpoint}/${aclBucket}`, '-v'], + httpCode => { assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, xml) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(xml.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].ID[0], ownerCanonicalId); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'FULL_CONTROL'); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/AllUsers'); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - done(); - }); - }); + done(); + }, + ); }); - it('should be able to create a bucket with a specific ACL', done => { - provideRawOutput([ - '--createBucket', - '--', - '-H', - 'x-amz-grant-read:uri=' + - 'http://acs.amazonaws.com/groups/global/AllUsers', - `${endpoint}/${aclBucket}2`, - '-v', - ], httpCode => { + it('should be able to get a canned ACL', done => { + provideRawOutput(['--', `${endpoint}/${aclBucket}?acl`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); - done(); + parseString(rawOutput.stdout, (err, xml) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(xml.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + xml.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + ownerCanonicalId, + ); + assert.strictEqual(xml.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], 'FULL_CONTROL'); + assert.strictEqual( + xml.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/AllUsers', + ); + assert.strictEqual(xml.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + done(); + }); }); }); - it('should be able to get a specifically set ACL', done => { + it('should be able to create a bucket with a specific ACL', done => { provideRawOutput( - ['--', `${endpoint}/${aclBucket}2?acl`, '-v'], - (httpCode, rawOutput) => { + [ + '--createBucket', + '--', + '-H', + 'x-amz-grant-read:uri=' + 'http://acs.amazonaws.com/groups/global/AllUsers', + `${endpoint}/${aclBucket}2`, + '-v', + ], + httpCode => { assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, xml) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(xml.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/AllUsers'); - assert.strictEqual(xml.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'READ'); - done(); - }); + done(); + }, + ); + }); + + it('should be able to get a specifically set ACL', done => { + provideRawOutput(['--', `${endpoint}/${aclBucket}2?acl`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, xml) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(xml.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + xml.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/AllUsers', + ); + assert.strictEqual(xml.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], 'READ'); + done(); }); + }); }); }); describe('s3curl getService', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + provideRawOutput(['--createBucket', '--', `${endpoint}/${aclBucket}`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); - provideRawOutput( - ['--createBucket', '--', `${endpoint}/${aclBucket}`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + done(); }); + }); }); after(done => { - deleteRemoteItems([ - bucketPath, - `${endpoint}/${aclBucket}`, - ], done); + deleteRemoteItems([bucketPath, `${endpoint}/${aclBucket}`], done); }); it('should get a list of all buckets created by user account', done => { - provideRawOutput( - ['--', `${endpoint}`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, xml) => { - if (err) { - assert.ifError(err); - } - const bucketNames = xml.ListAllMyBucketsResult - .Buckets[0].Bucket - .map(item => item.Name[0]); - const whereIsMyBucket = bucketNames.indexOf(bucket); - assert(whereIsMyBucket > -1); - const whereIsMyAclBucket = bucketNames.indexOf(aclBucket); - assert(whereIsMyAclBucket > -1); - done(); - }); + provideRawOutput(['--', `${endpoint}`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, xml) => { + if (err) { + assert.ifError(err); + } + const bucketNames = xml.ListAllMyBucketsResult.Buckets[0].Bucket.map(item => item.Name[0]); + const whereIsMyBucket = bucketNames.indexOf(bucket); + assert(whereIsMyBucket > -1); + const whereIsMyAclBucket = bucketNames.indexOf(aclBucket); + assert(whereIsMyAclBucket > -1); + done(); }); + }); }); }); describe('s3curl putObject', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - createFile(upload, 1048576, done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + createFile(upload, 1048576, done); + }); }); after(done => { - deleteRemoteItems([ - `${prefixedPath}${upload}1`, - `${prefixedPath}${upload}2`, - `${prefixedPath}${upload}3`, - bucketPath, - ], done); + deleteRemoteItems( + [`${prefixedPath}${upload}1`, `${prefixedPath}${upload}2`, `${prefixedPath}${upload}3`, bucketPath], + done, + ); }); // curl behavior is not consistent across the environments // skipping the test for now - it.skip('should not be able to put an object if request does not have ' + - 'content-length header', - done => { - provideRawOutput([ - '--debug', - `--put=${upload}`, - '--', - '-H', - 'content-length:', - `${prefixedPath}${upload}1`, - '-v', - ], (httpCode, rawOutput) => { + it.skip('should not be able to put an object if request does not have ' + 'content-length header', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', '-H', 'content-length:', `${prefixedPath}${upload}1`, '-v'], + (httpCode, rawOutput) => { assert.strictEqual(httpCode, '411 LENGTH REQUIRED'); assertError(rawOutput.stdout, 'MissingContentLength', done); - }); - }); - - it('should not be able to put an object if content-md5 header is ' + - 'invalid', - done => { - provideRawOutput(['--debug', `--put=${upload}`, - '--contentMd5', 'toto', '--', - `${endpoint}/${bucket}/` + - `${prefix}${delimiter}${upload}1`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidDigest', done); - }); - }); - - // skip until we figure out how to parse the response in the CI - it.skip('should not be able to put an object if content-md5 header is ' + - 'mismatched MD5', - done => { - provideRawOutput(['--debug', `--put=${upload}`, - '--contentMd5', 'rL0Y20zC+Fzt72VPzMSk2A==', '--', - `${endpoint}/${bucket}/` + - `${prefix}${delimiter}${upload}1`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'BadDigest', done); - }); - }); + }, + ); + }); - it('should not be able to put an object if using streaming ' + - 'chunked-upload with a valid V2 signature', - done => { - provideRawOutput([ + it('should not be able to put an object if content-md5 header is ' + 'invalid', done => { + provideRawOutput( + [ '--debug', `--put=${upload}`, + '--contentMd5', + 'toto', '--', - '-H', - 'x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD', - `${endpoint}/${bucket}/${prefix}${delimiter}${upload}1`, - '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidArgument', done); - }); - }); + `${endpoint}/${bucket}/` + `${prefix}${delimiter}${upload}1`, + '-v', + ], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidDigest', done); + }, + ); + }); - it('should not be able to put an object in a bucket with an invalid name', - done => { - provideRawOutput([ + // skip until we figure out how to parse the response in the CI + it.skip('should not be able to put an object if content-md5 header is ' + 'mismatched MD5', done => { + provideRawOutput( + [ '--debug', `--put=${upload}`, + '--contentMd5', + 'rL0Y20zC+Fzt72VPzMSk2A==', '--', - `${endpoint}/2/${basePath}${upload}1`, + `${endpoint}/${bucket}/` + `${prefix}${delimiter}${upload}1`, '-v', - ], (httpCode, rawOutput) => { + ], + (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidBucketName', done); - }); - }); + assertError(rawOutput.stdout, 'BadDigest', done); + }, + ); + }); - it('should not be able to put an object in a bucket that does not exist', - done => { - provideRawOutput([ + it('should not be able to put an object if using streaming ' + 'chunked-upload with a valid V2 signature', done => { + provideRawOutput( + [ '--debug', `--put=${upload}`, '--', - `${endpoint}/${nonexist}/${basePath}${upload}1`, + '-H', + 'x-amz-content-sha256: STREAMING-AWS4-HMAC-SHA256-PAYLOAD', + `${endpoint}/${bucket}/${prefix}${delimiter}${upload}1`, '-v', - ], (httpCode, rawOutput) => { + ], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidArgument', done); + }, + ); + }); + + it('should not be able to put an object in a bucket with an invalid name', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', `${endpoint}/2/${basePath}${upload}1`, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidBucketName', done); + }, + ); + }); + + it('should not be able to put an object in a bucket that does not exist', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', `${endpoint}/${nonexist}/${basePath}${upload}1`, '-v'], + (httpCode, rawOutput) => { assert.strictEqual(httpCode, '404 NOT FOUND'); assertError(rawOutput.stdout, 'NoSuchBucket', done); - }); - }); + }, + ); + }); - it('should not be able to put an object in a bucket with an empty name', - done => { - provideRawOutput([ - '--debug', - `--put=${upload}`, - '--', - `${endpoint}//${basePath}/${upload}1`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); - done(); - }); + it('should not be able to put an object in a bucket with an empty name', done => { + provideRawOutput( + ['--debug', `--put=${upload}`, '--', `${endpoint}//${basePath}/${upload}1`, '-v'], + httpCode => { + assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); + done(); + }, + ); }); - it('should put first object in existing bucket with prefix ' + - 'and delimiter', done => { - provideRawOutput([ - '--debug', - `--put=${upload}`, - '--', - `${prefixedPath}${upload}1`, - '-v', - ], httpCode => { + it('should put first object in existing bucket with prefix ' + 'and delimiter', done => { + provideRawOutput(['--debug', `--put=${upload}`, '--', `${prefixedPath}${upload}1`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); }); }); - it('should put second object in existing bucket with prefix ' + - 'and delimiter', done => { - provideRawOutput( - [`--put=${upload}`, '--', `${prefixedPath}${upload}2`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + it('should put second object in existing bucket with prefix ' + 'and delimiter', done => { + provideRawOutput([`--put=${upload}`, '--', `${prefixedPath}${upload}2`, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); - it('should put third object in existing bucket with prefix ' + - 'and delimiter', done => { - provideRawOutput([ - `--put=${upload}`, - '--', - `${prefixedPath}${upload}3`, - '-v', - ], httpCode => { + it('should put third object in existing bucket with prefix ' + 'and delimiter', done => { + provideRawOutput([`--put=${upload}`, '--', `${prefixedPath}${upload}3`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); }); @@ -581,21 +496,15 @@ describe('s3curl putObject', () => { }); describe('s3curl getBucket', () => { - const objects = [ - `${prefixedPath}${upload}1`, - `${prefixedPath}${upload}2`, - `${prefixedPath}${upload}3`, - ]; + const objects = [`${prefixedPath}${upload}1`, `${prefixedPath}${upload}2`, `${prefixedPath}${upload}3`]; before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - createFile(upload, 1048576, () => { - putObjects(upload, objects, done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + createFile(upload, 1048576, () => { + putObjects(upload, objects, done); }); + }); }); after(done => { @@ -604,143 +513,114 @@ describe('s3curl getBucket', () => { }); it('should list all objects if no prefix or delimiter specified', done => { - provideRawOutput( - ['--', bucketPath, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.ListBucketResult - .Contents[0].Key[0], `${basePath}${upload}1`); - assert.strictEqual(result.ListBucketResult - .Contents[1].Key[0], `${basePath}${upload}2`); - assert.strictEqual(result.ListBucketResult - .Contents[2].Key[0], `${basePath}${upload}3`); - done(); - }); - }); - }); - - it('should list a common prefix if a common prefix and delimiter are ' + - 'specified', done => { - provideRawOutput([ - '--', - `${bucketPath}?delimiter=${delimiter}&prefix=${prefix}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', bucketPath, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - assert.strictEqual(result.ListBucketResult - .CommonPrefixes[0].Prefix[0], basePath); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], `${basePath}${upload}1`); + assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], `${basePath}${upload}2`); + assert.strictEqual(result.ListBucketResult.Contents[2].Key[0], `${basePath}${upload}3`); done(); }); }); }); - it('should not list a common prefix if no delimiter is specified', done => { + it('should list a common prefix if a common prefix and delimiter are ' + 'specified', done => { provideRawOutput( - ['--', `${bucketPath}?&prefix=${prefix}`, '-v'], + ['--', `${bucketPath}?delimiter=${delimiter}&prefix=${prefix}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - const keys = Object.keys(result.ListBucketResult); - const location = keys.indexOf('CommonPrefixes'); - assert.strictEqual(location, -1); - assert.strictEqual(result.ListBucketResult - .Contents[0].Key[0], `${basePath}${upload}1`); + assert.strictEqual(result.ListBucketResult.CommonPrefixes[0].Prefix[0], basePath); done(); }); + }, + ); + }); + + it('should not list a common prefix if no delimiter is specified', done => { + provideRawOutput(['--', `${bucketPath}?&prefix=${prefix}`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + const keys = Object.keys(result.ListBucketResult); + const location = keys.indexOf('CommonPrefixes'); + assert.strictEqual(location, -1); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], `${basePath}${upload}1`); + done(); }); + }); }); - it('should provide a next marker if maxs keys exceeded ' + - 'and delimiter specified', done => { - provideRawOutput( - ['--', `${bucketPath}?delimiter=x&max-keys=2`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.ListBucketResult - .NextMarker[0], `${basePath}${upload}2`); - assert.strictEqual(result.ListBucketResult - .IsTruncated[0], 'true'); - done(); - }); + it('should provide a next marker if maxs keys exceeded ' + 'and delimiter specified', done => { + provideRawOutput(['--', `${bucketPath}?delimiter=x&max-keys=2`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(result.ListBucketResult.NextMarker[0], `${basePath}${upload}2`); + assert.strictEqual(result.ListBucketResult.IsTruncated[0], 'true'); + done(); }); + }); }); it('should return InvalidArgument error with negative max-keys', done => { - provideRawOutput( - ['--', `${bucketPath}?&max-keys=-2`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidArgument', done); - }); + provideRawOutput(['--', `${bucketPath}?&max-keys=-2`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidArgument', done); + }); }); it('should return InvalidArgument error with invalid max-keys', done => { - provideRawOutput( - ['--', `${bucketPath}?max-keys='slash'`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidArgument', done); - }); + provideRawOutput(['--', `${bucketPath}?max-keys='slash'`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidArgument', done); + }); }); it('should return an EncodingType XML tag with the value "url"', done => { - provideRawOutput( - ['--', bucketPath, '-G', '-d', 'encoding-type=url', '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.ListBucketResult - .EncodingType[0], 'url'); - done(); - }); + provideRawOutput(['--', bucketPath, '-G', '-d', 'encoding-type=url', '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(result.ListBucketResult.EncodingType[0], 'url'); + done(); }); + }); }); - it('should return an InvalidArgument error when given an invalid ' + - 'encoding type', done => { - provideRawOutput( - ['--', bucketPath, '-G', '-d', 'encoding-type=invalidURI', '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - assert.strictEqual(result.Error.Code[0], 'InvalidArgument'); - assert.strictEqual(result.Error.Message[0], - 'Invalid Encoding Method specified in Request'); - done(); - }); + it('should return an InvalidArgument error when given an invalid ' + 'encoding type', done => { + provideRawOutput(['--', bucketPath, '-G', '-d', 'encoding-type=invalidURI', '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + assert.strictEqual(result.Error.Code[0], 'InvalidArgument'); + assert.strictEqual(result.Error.Message[0], 'Invalid Encoding Method specified in Request'); + done(); }); + }); }); }); describe('s3curl head bucket', () => { before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); after(done => { @@ -748,78 +628,59 @@ describe('s3curl head bucket', () => { }); it('should return a 404 response if bucket does not exist', done => { - provideRawOutput( - ['--head', '--', `${endpoint}/${nonexist}`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '404 NOT FOUND'); - done(); - }); + provideRawOutput(['--head', '--', `${endpoint}/${nonexist}`, '-v'], httpCode => { + assert.strictEqual(httpCode, '404 NOT FOUND'); + done(); + }); }); - it('should return a 200 response if bucket exists' + - ' and user is authorized', done => { - provideRawOutput( - ['--head', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + it('should return a 200 response if bucket exists' + ' and user is authorized', done => { + provideRawOutput(['--head', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); describe('s3curl getObject', () => { before(done => { createFile(upload, 1048576, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); after('delete created file and downloaded file', done => { - const objects = [ - `${bucketPath}/getter`, - bucketPath, - ]; + const objects = [`${bucketPath}/getter`, bucketPath]; deleteRemoteItems(objects, () => { deleteFile(upload, () => deleteFile(download, done)); }); }); it('should put object with metadata', done => { - provideRawOutput([ - `--put=${upload}`, - '--', - '-H', - 'x-amz-meta-mine:BestestObjectEver', - `${bucketPath}/getter`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); - }); - - it('should get an existing file in an existing bucket', done => { provideRawOutput( - ['--', '-o', download, `${bucketPath}/getter`, '-v'], + [`--put=${upload}`, '--', '-H', 'x-amz-meta-mine:BestestObjectEver', `${bucketPath}/getter`, '-v'], httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); - }); + }, + ); }); - it('should return an error if getting object with empty bucket name', - done => { - provideRawOutput( - ['--', '-o', download, `${endpoint}//getter`, '-v'], - httpCode => { - assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); - done(); - }); + it('should get an existing file in an existing bucket', done => { + provideRawOutput(['--', '-o', download, `${bucketPath}/getter`, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); + }); + + it('should return an error if getting object with empty bucket name', done => { + provideRawOutput(['--', '-o', download, `${endpoint}//getter`, '-v'], httpCode => { + assert.strictEqual(httpCode, '405 METHOD NOT ALLOWED'); + done(); + }); }); it.skip('downloaded file should equal uploaded file', done => { @@ -830,160 +691,135 @@ describe('s3curl getObject', () => { describe('s3curl head object', () => { before(done => { createFile(upload, 1048576, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - provideRawOutput([ - `--put=${upload}`, - '--', - '-H', - 'x-amz-meta-mine:BestestObjectEver', - `${bucketPath}/getter`, - '-v', - ], httpCode => { + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + provideRawOutput( + [`--put=${upload}`, '--', '-H', 'x-amz-meta-mine:BestestObjectEver', `${bucketPath}/getter`, '-v'], + httpCode => { assert.strictEqual(httpCode, '200 OK'); done(); - }); - }); + }, + ); + }); }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/getter`, - bucketPath, - ], done); + deleteRemoteItems([`${bucketPath}/getter`, bucketPath], done); }); it("should get object's metadata", done => { - provideRawOutput( - ['--head', '--', `${bucketPath}/getter`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '200 OK'); - const lines = rawOutput.stdout.split('\n'); - const userMetadata = 'x-amz-meta-mine: BestestObjectEver\r'; - assert(lines.indexOf(userMetadata) > -1); - assert(rawOutput.stdout.indexOf('ETag') > -1); - done(); - }); + provideRawOutput(['--head', '--', `${bucketPath}/getter`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '200 OK'); + const lines = rawOutput.stdout.split('\n'); + const userMetadata = 'x-amz-meta-mine: BestestObjectEver\r'; + assert(lines.indexOf(userMetadata) > -1); + assert(rawOutput.stdout.indexOf('ETag') > -1); + done(); + }); }); }); describe('s3curl object ACLs', () => { before(done => { createFile(aclUpload, 512000, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }); }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/${aclUpload}withcannedacl`, - `${bucketPath}/${aclUpload}withspecificacl`, - bucketPath, - ], () => deleteFile(aclUpload, done)); + deleteRemoteItems( + [`${bucketPath}/${aclUpload}withcannedacl`, `${bucketPath}/${aclUpload}withspecificacl`, bucketPath], + () => deleteFile(aclUpload, done), + ); }); it('should put an object with a canned ACL', done => { - provideRawOutput([ - `--put=${aclUpload}`, - '--', - '-H', - 'x-amz-acl:public-read', - `${bucketPath}/${aclUpload}withcannedacl`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput( + [ + `--put=${aclUpload}`, + '--', + '-H', + 'x-amz-acl:public-read', + `${bucketPath}/${aclUpload}withcannedacl`, + '-v', + ], + httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }, + ); }); it("should get an object's canned ACL", done => { - provideRawOutput([ - '--', - `${bucketPath}/${aclUpload}withcannedacl?acl`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}/${aclUpload}withcannedacl?acl`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - assert.strictEqual(result.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].ID[0], ownerCanonicalId); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/AllUsers'); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[1] - .Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + ownerCanonicalId, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/AllUsers', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); done(); }); }); }); it('should put an object with a specific ACL', done => { - provideRawOutput([ - `--put=${aclUpload}`, - '--', - '-H', - 'x-amz-grant-read:uri=' + - 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers', - `${bucketPath}/${aclUpload}withspecificacl`, - '-v', - ], httpCode => { - assert.strictEqual(httpCode, '200 OK'); - done(); - }); + provideRawOutput( + [ + `--put=${aclUpload}`, + '--', + '-H', + 'x-amz-grant-read:uri=' + 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers', + `${bucketPath}/${aclUpload}withspecificacl`, + '-v', + ], + httpCode => { + assert.strictEqual(httpCode, '200 OK'); + done(); + }, + ); }); it("should get an object's specific ACL", done => { - provideRawOutput([ - '--', - `${bucketPath}/${aclUpload}withspecificacl?acl`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}/${aclUpload}withspecificacl?acl`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { if (err) { assert.ifError(err); } - assert.strictEqual(result.AccessControlPolicy - .Owner[0].ID[0], ownerCanonicalId); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Grantee[0].URI[0], - 'http://acs.amazonaws.com/groups/global/' + - 'AuthenticatedUsers'); - assert.strictEqual(result.AccessControlPolicy - .AccessControlList[0].Grant[0] - .Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.Owner[0].ID[0], ownerCanonicalId); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].URI[0], + 'http://acs.amazonaws.com/groups/global/' + 'AuthenticatedUsers', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], 'READ'); done(); }); }); }); - it('should return a NoSuchKey error if try to get an object' + - 'ACL for an object that does not exist', done => { - provideRawOutput( - ['--', `${bucketPath}/keydoesnotexist?acl`, '-v'], - (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '404 NOT FOUND'); - assertError(rawOutput.stdout, 'NoSuchKey', done); - }); + it('should return a NoSuchKey error if try to get an object' + 'ACL for an object that does not exist', done => { + provideRawOutput(['--', `${bucketPath}/keydoesnotexist?acl`, '-v'], (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '404 NOT FOUND'); + assertError(rawOutput.stdout, 'NoSuchKey', done); + }); }); }); @@ -993,104 +829,70 @@ describe('s3curl multipart upload', () => { let uploadId = null; before(done => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - // initiate mpu - provideRawOutput([ - '--', - '-X', - 'POST', - `${bucketPath}/${key}?uploads`, - '-v', - ], (httpCode, rawOutput) => { - parseString(rawOutput.stdout, (err, result) => { - if (err) { - assert.ifError(err); - } - uploadId = - result.InitiateMultipartUploadResult.UploadId[0]; - // create file to copy - createFile(upload, 100, () => { - // put file to copy - putObjects(upload, [`${bucketPath}/copyme`], done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + // initiate mpu + provideRawOutput(['--', '-X', 'POST', `${bucketPath}/${key}?uploads`, '-v'], (httpCode, rawOutput) => { + parseString(rawOutput.stdout, (err, result) => { + if (err) { + assert.ifError(err); + } + uploadId = result.InitiateMultipartUploadResult.UploadId[0]; + // create file to copy + createFile(upload, 100, () => { + // put file to copy + putObjects(upload, [`${bucketPath}/copyme`], done); }); }); }); + }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/copyme`, - `${bucketPath}/${key}?uploadId=${uploadId}`, - bucketPath, - ], () => deleteFile(upload, done)); + deleteRemoteItems([`${bucketPath}/copyme`, `${bucketPath}/${key}?uploadId=${uploadId}`, bucketPath], () => + deleteFile(upload, done), + ); }); it('should return error for list parts call if no key sent', done => { - provideRawOutput([ - '--', - `${bucketPath}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); assertError(rawOutput.stdout, 'InvalidRequest', done); }); }); it('should return error for put part call if no key sent', done => { - provideRawOutput([ - '--', - '-X', 'PUT', - `${bucketPath}?partNumber=1&uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { - assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'InvalidRequest', done); - }); + provideRawOutput( + ['--', '-X', 'PUT', `${bucketPath}?partNumber=1&uploadId=${uploadId}`, '-v'], + (httpCode, rawOutput) => { + assert.strictEqual(httpCode, '400 BAD REQUEST'); + assertError(rawOutput.stdout, 'InvalidRequest', done); + }, + ); }); it('should return error for complete mpu call if no key sent', done => { - provideRawOutput([ - '--', - '-X', 'POST', - `${bucketPath}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', '-X', 'POST', `${bucketPath}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); assertError(rawOutput.stdout, 'InvalidRequest', done); }); }); it('should return error for abort mpu call if no key sent', done => { - provideRawOutput([ - '--', - '-X', 'DELETE', - `${bucketPath}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', '-X', 'DELETE', `${bucketPath}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); assertError(rawOutput.stdout, 'InvalidRequest', done); }); }); it('should list parts of multipart upload with no parts', done => { - provideRawOutput([ - '--', - `${bucketPath}/${key}?uploadId=${uploadId}`, - '-v', - ], (httpCode, rawOutput) => { + provideRawOutput(['--', `${bucketPath}/${key}?uploadId=${uploadId}`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { - assert.strictEqual(result.ListPartsResult.UploadId[0], - uploadId); - assert.strictEqual(result.ListPartsResult.Bucket[0], - bucket); + assert.strictEqual(result.ListPartsResult.UploadId[0], uploadId); + assert.strictEqual(result.ListPartsResult.Bucket[0], bucket); assert.strictEqual(result.ListPartsResult.Key[0], key); - assert.strictEqual(result.ListPartsResult.Part, - undefined); + assert.strictEqual(result.ListPartsResult.Part, undefined); done(); }); }); @@ -1098,68 +900,76 @@ describe('s3curl multipart upload', () => { it('should copy a part and return lastModified as ISO', done => { provideRawOutput( - ['--', `${bucketPath}/${key}?uploadId=${uploadId}&partNumber=1`, - '-X', 'PUT', '-H', - `x-amz-copy-source:${bucket}/copyme`, '-v'], + [ + '--', + `${bucketPath}/${key}?uploadId=${uploadId}&partNumber=1`, + '-X', + 'PUT', + '-H', + `x-amz-copy-source:${bucket}/copyme`, + '-v', + ], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { - const lastModified = result.CopyPartResult - .LastModified[0]; + const lastModified = result.CopyPartResult.LastModified[0]; const isoDateString = new Date(lastModified).toISOString(); assert.strictEqual(lastModified, isoDateString); done(); }); - }); + }, + ); }); }); describe('s3curl copy object', () => { before(done => { createFile(upload, 1048576, () => { - provideRawOutput( - ['--createBucket', '--', bucketPath, '-v'], - httpCode => { - assert.strictEqual(httpCode, '200 OK'); - putObjects(upload, [`${bucketPath}/copyme`], done); - }); + provideRawOutput(['--createBucket', '--', bucketPath, '-v'], httpCode => { + assert.strictEqual(httpCode, '200 OK'); + putObjects(upload, [`${bucketPath}/copyme`], done); + }); }); }); after(done => { - deleteRemoteItems([ - `${bucketPath}/copyme`, - `${bucketPath}/iamacopy`, - bucketPath, - ], () => deleteFile(upload, done)); + deleteRemoteItems([`${bucketPath}/copyme`, `${bucketPath}/iamacopy`, bucketPath], () => + deleteFile(upload, done), + ); }); it('should copy an object and return lastModified as ISO', done => { provideRawOutput( - ['--', `${bucketPath}/iamacopy`, '-X', 'PUT', '-H', - `x-amz-copy-source:${bucket}/copyme`, '-v'], + ['--', `${bucketPath}/iamacopy`, '-X', 'PUT', '-H', `x-amz-copy-source:${bucket}/copyme`, '-v'], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '200 OK'); parseString(rawOutput.stdout, (err, result) => { - const lastModified = result.CopyObjectResult - .LastModified[0]; + const lastModified = result.CopyObjectResult.LastModified[0]; const isoDateString = new Date(lastModified).toISOString(); assert.strictEqual(lastModified, isoDateString); done(); }); - }); + }, + ); }); }); describe('s3curl multi-object delete', () => { it('should return an error if md5 is wrong', done => { - provideRawOutput(['--post', 'multiDelete.xml', '--contentMd5', - 'p5/WA/oEr30qrEEl21PAqw==', '--', - `${endpoint}/${bucket}/?delete`, '-v'], + provideRawOutput( + [ + '--post', + 'multiDelete.xml', + '--contentMd5', + 'p5/WA/oEr30qrEEl21PAqw==', + '--', + `${endpoint}/${bucket}/?delete`, + '-v', + ], (httpCode, rawOutput) => { assert.strictEqual(httpCode, '400 BAD REQUEST'); - assertError(rawOutput.stdout, 'BadDigest', - done); - }); + assertError(rawOutput.stdout, 'BadDigest', done); + }, + ); }); }); diff --git a/tests/functional/sse-kms-migration/arnPrefix.js b/tests/functional/sse-kms-migration/arnPrefix.js index 5a70ed5e97..60231be12a 100644 --- a/tests/functional/sse-kms-migration/arnPrefix.js +++ b/tests/functional/sse-kms-migration/arnPrefix.js @@ -30,12 +30,10 @@ describe('SSE KMS arnPrefix', () => { bkts[bktConf.name] = bkt; if (bktConf.algo && bktConf.masterKeyId) { bkt.kmsKeyInfo = await helpers.createKmsKey(log); - bkt.kmsKey = bktConf.arnPrefix - ? bkt.kmsKeyInfo.masterKeyArn - : bkt.kmsKeyInfo.masterKeyId; + bkt.kmsKey = bktConf.arnPrefix ? bkt.kmsKeyInfo.masterKeyArn : bkt.kmsKeyInfo.masterKeyId; } - await helpers.s3.createBucket(({ Bucket: bkt.name })); - await helpers.s3.createBucket(({ Bucket: bkt.vname })); + await helpers.s3.createBucket({ Bucket: bkt.name }); + await helpers.s3.createBucket({ Bucket: bkt.vname }); if (bktConf.deleteSSE) { await scenarios.deleteBucketSSEBeforeEach(bkt.name, log); await scenarios.deleteBucketSSEBeforeEach(bkt.vname, log); @@ -45,38 +43,44 @@ describe('SSE KMS arnPrefix', () => { await helpers.s3.putBucketEncryption({ Bucket: bkt.name, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); await helpers.s3.putBucketEncryption({ Bucket: bkt.vname, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); } // Put an object for each SSE conf in each bucket - await Promise.all(scenarios.testCases.map(async objConf => { - const obj = { - name: `for-copy-enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(for-copy-enc-obj-${objConf.name})`, - }; - bkt.objs[objConf.name] = obj; - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } + await Promise.all( + scenarios.testCases.map(async objConf => { + const obj = { + name: `for-copy-enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(for-copy-enc-obj-${objConf.name})`, + }; + bkt.objs[objConf.name] = obj; + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } - return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); - })); + return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); + }), + ); }; before('setup', async () => { - console.log('Run arnPrefix', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run arnPrefix', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = (await helpers.s3.listBuckets()).Buckets.map(b => b.Name); console.log('List buckets:', allBuckets); await helpers.MD.setup(); @@ -85,15 +89,19 @@ describe('SSE KMS arnPrefix', () => { // pre cleanup await helpers.cleanup(copyBkt); await helpers.cleanup(mpuCopyBkt); - await Promise.all(Object.values(bkts).map(async bkt => { - await helpers.cleanup(bkt.name); - return await helpers.cleanup(bkt.vname); - })); - } catch (e) { void e; } + await Promise.all( + Object.values(bkts).map(async bkt => { + await helpers.cleanup(bkt.name); + return await helpers.cleanup(bkt.vname); + }), + ); + } catch (e) { + void e; + } // init copy bucket - await helpers.s3.createBucket(({ Bucket: copyBkt })); - await helpers.s3.createBucket(({ Bucket: mpuCopyBkt })); + await helpers.s3.createBucket({ Bucket: copyBkt }); + await helpers.s3.createBucket({ Bucket: mpuCopyBkt }); await helpers.s3.putBucketEncryption({ Bucket: copyBkt, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'aws:kms', masterKeyId: copyKmsKey }), @@ -108,259 +116,353 @@ describe('SSE KMS arnPrefix', () => { await helpers.cleanup(copyBkt); await helpers.cleanup(mpuCopyBkt); // Clean every bucket - await Promise.all(Object.values(bkts).map(async bkt => { - await helpers.cleanup(bkt.name); - return await helpers.cleanup(bkt.vname); - })); + await Promise.all( + Object.values(bkts).map(async bkt => { + await helpers.cleanup(bkt.name); + return await helpers.cleanup(bkt.vname); + }), + ); }); - scenarios.testCases.forEach(bktConf => describe(`bucket enc-bkt-${bktConf.name}`, () => { - let bkt = bkts[bktConf.name]; - - before(() => { - bkt = bkts[bktConf.name]; - }); + scenarios.testCases.forEach(bktConf => + describe(`bucket enc-bkt-${bktConf.name}`, () => { + let bkt = bkts[bktConf.name]; - if (bktConf.deleteSSE) { - beforeEach(async () => { - await scenarios.deleteBucketSSEBeforeEach(bkt.name, log); - await scenarios.deleteBucketSSEBeforeEach(bkt.vname, log); + before(() => { + bkt = bkts[bktConf.name]; }); - } - if (!bktConf.algo) { - if (!bktConf.deleteSSE && helpers.config.globalEncryptionEnabled) { - it('GetBucketEncryption should return AES256 because of globalEncryptionEnabled', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, 'AES256', null, 'after')); - } else { - it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', - async () => await scenarios.tests.getBucketSSEError(bkt.name)); - if (!bktConf.deleteSSE) { - it('should have non mandatory SSE in bucket MD as test init put an object with AES256', - async () => await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'after')); - } + if (bktConf.deleteSSE) { + beforeEach(async () => { + await scenarios.deleteBucketSSEBeforeEach(bkt.name, log); + await scenarios.deleteBucketSSEBeforeEach(bkt.vname, log); + }); } - } else { - it('GetBucketEncryption should return SSE with arnPrefix to key', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, bktConf.algo, - bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'after')); - } - scenarios.testCasesObj.forEach(objConf => it(`should assert uploaded objects with SSE ${objConf.name}`, - async () => scenarios.tests.getPreUploadedObject(bkt.name, - { objConf, obj: bkt.objs[objConf.name] }, { bktConf, bkt }, 'after'))); - - scenarios.testCasesObj.forEach(objConf => describe(`object enc-obj-${objConf.name}`, () => { - const obj = { - name: `enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(enc-obj-${objConf.name})`, - }; - /** to be used as source of copy */ - let objForCopy; - - before(async () => { - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; + if (!bktConf.algo) { + if (!bktConf.deleteSSE && helpers.config.globalEncryptionEnabled) { + it('GetBucketEncryption should return AES256 because of globalEncryptionEnabled', async () => + await scenarios.tests.getBucketSSE(bkt.name, log, 'AES256', null, 'after')); + } else { + it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', async () => + await scenarios.tests.getBucketSSEError(bkt.name)); + if (!bktConf.deleteSSE) { + it('should have non mandatory SSE in bucket MD as test init put an object with AES256', async () => + await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'after')); + } } - objForCopy = bkt.objs[objConf.name]; - }); + } else { + it('GetBucketEncryption should return SSE with arnPrefix to key', async () => + await scenarios.tests.getBucketSSE( + bkt.name, + log, + bktConf.algo, + bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, + 'after', + )); + } - it(`should PutObject ${obj.name} overriding bucket SSE`, - async () => scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt }, 'after')); - - // CopyObject scenarios - [ - { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, - { name: `${obj.name} into same bucket with object SSE config` }, - { name: `from encrypted source into ${obj.name} with object SSE config` }, - ].forEach(({ name, forceBktSSE }, index) => - it(`should CopyObject ${name}`, async () => - await scenarios.tests.copyObjectAndSSE( - { copyBkt, objForCopy, copyObj }, - { objConf, obj }, + scenarios.testCasesObj.forEach(objConf => + it(`should assert uploaded objects with SSE ${objConf.name}`, async () => + scenarios.tests.getPreUploadedObject( + bkt.name, + { objConf, obj: bkt.objs[objConf.name] }, { bktConf, bkt }, - { index, forceBktSSE }, 'after', - ))); - - // after SSE migration implementation all mpu with sse are fixed - it('should encrypt MPU and put 2 encrypted parts', async () => { - const mpuKey = `${obj.name}-mpu`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; - const newParts = []; - for (const [index, body] of partsBody.entries()) { - const part = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: body, - Key: mpuKey, - PartNumber: index + 1, - }, mpu, objConf.algo || bktConf.algo, 'after'); - newParts.push(part); - } - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'after'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `${obj.body}-MPU1${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); - }); - - it('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const part1 = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpu, objConf.algo || bktConf.algo, 'after'); - const part2 = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: 2, - }, mpu, objConf.algo || bktConf.algo, 'after'); - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts: [part1, part2] }, - mpu, objConf.algo || bktConf.algo, 'after'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `BODY(copy)${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); - }); - - it('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - const newParts = []; - for (const [index, range] of sourceRanges.entries()) { - const part = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpu, objConf.algo || bktConf.algo, 'after'); - newParts.push(part); - } - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'after'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: 'copyBODY', - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); - }); - - it(`should PutObject versioned with SSE ${obj.name}`, async () => { - // ensure versioned bucket is empty - await helpers.bucketUtil.empty(bkt.vname); - let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }); - // regularly count versioned objects - assert.strictEqual(Versions?.length, 0); - - const bodyBase = `BODY(${obj.name})-base`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); - const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyBase }, - { objConf, obj }, { bktConf, bkt }, {}, 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 1); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Enabled' }, - }); - - const bodyV1 = `BODY(${obj.name})-v1`; - const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); - const bodyV2 = `BODY(${obj.name})-v2`; - const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); - assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Suspended' }, - }); + )), + ); - // should be fine after version suspension - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - // put a new null version - const bodyFinal = `BODY(${obj.name})-final`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, - {}, 'after'); // null - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, - 'null', 'after'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - }); - })); - })); + scenarios.testCasesObj.forEach(objConf => + describe(`object enc-obj-${objConf.name}`, () => { + const obj = { + name: `enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(enc-obj-${objConf.name})`, + }; + /** to be used as source of copy */ + let objForCopy; + + before(async () => { + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + objForCopy = bkt.objs[objConf.name]; + }); + + it(`should PutObject ${obj.name} overriding bucket SSE`, async () => + scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt }, 'after')); + + // CopyObject scenarios + [ + { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, + { name: `${obj.name} into same bucket with object SSE config` }, + { name: `from encrypted source into ${obj.name} with object SSE config` }, + ].forEach(({ name, forceBktSSE }, index) => + it(`should CopyObject ${name}`, async () => + await scenarios.tests.copyObjectAndSSE( + { copyBkt, objForCopy, copyObj }, + { objConf, obj }, + { bktConf, bkt }, + { index, forceBktSSE }, + 'after', + )), + ); + + // after SSE migration implementation all mpu with sse are fixed + it('should encrypt MPU and put 2 encrypted parts', async () => { + const mpuKey = `${obj.name}-mpu`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; + const newParts = []; + for (const [index, body] of partsBody.entries()) { + const part = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: index + 1, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + newParts.push(part); + } + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `${obj.body}-MPU1${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); + }); + + it('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { + const mpuKey = `${obj.name}-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const part1 = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const part2 = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: 2, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts: [part1, part2] }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `BODY(copy)${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); + }); + + it('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => { + const mpuKey = `${obj.name}-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + const newParts = []; + for (const [index, range] of sourceRanges.entries()) { + const part = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + newParts.push(part); + } + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'after', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: 'copyBODY', + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, {}, 'after'); + }); + + it(`should PutObject versioned with SSE ${obj.name}`, async () => { + // ensure versioned bucket is empty + await helpers.bucketUtil.empty(bkt.vname); + let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }); + // regularly count versioned objects + assert.strictEqual(Versions?.length, 0); + + const bodyBase = `BODY(${obj.name})-base`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); + const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 1); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Enabled' }, + }); + + const bodyV1 = `BODY(${obj.name})-v1`; + const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); + const bodyV2 = `BODY(${obj.name})-v2`; + const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); + assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Suspended' }, + }); + + // should be fine after version suspension + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + // put a new null version + const bodyFinal = `BODY(${obj.name})-final`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + {}, + 'after', + ); // null + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + 'null', + 'after', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + }); + }), + ); + }), + ); it('should encrypt MPU and copy parts from every buckets and objects matrice', async () => { await helpers.s3.putBucketEncryption({ @@ -369,8 +471,7 @@ describe('SSE KMS arnPrefix', () => { ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }), }); const mpuKey = 'mpucopy'; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); + const mpu = await helpers.s3.createMultipartUpload(helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); const copyPartArg = { UploadId: mpu.UploadId, Bucket: mpuCopyBkt, @@ -380,18 +481,20 @@ describe('SSE KMS arnPrefix', () => { const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { const bkt = bkts[bktConf.name]; - return acc.concat(scenarios.testCasesObj.map(async (objConf, objIdx) => { - const obj = bkt.objs[objConf.name]; + return acc.concat( + scenarios.testCasesObj.map(async (objConf, objIdx) => { + const obj = bkt.objs[objConf.name]; - const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; - const res = await helpers.s3.uploadPartCopy({ - ...copyPartArg, - PartNumber: partNumber, - CopySource: `${bkt.name}/${obj.name}`, - }); + const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; + const res = await helpers.s3.uploadPartCopy({ + ...copyPartArg, + PartNumber: partNumber, + CopySource: `${bkt.name}/${obj.name}`, + }); - return { partNumber, body: obj.body, res: res.CopyPartResult }; - })); + return { partNumber, body: obj.body, res: res.CopyPartResult }; + }), + ); }, []); const parts = await Promise.all(uploadPromises); @@ -409,8 +512,13 @@ describe('SSE KMS arnPrefix', () => { Key: mpuKey, Body: parts.reduce((acc, part) => `${acc}${part.body}`, ''), }; - await scenarios.assertObjectSSE(assertion, { objConf: {}, obj: {} }, - { bktConf: { algo: 'AES256' }, bkt: {} }, {}, 'after'); + await scenarios.assertObjectSSE( + assertion, + { objConf: {}, obj: {} }, + { bktConf: { algo: 'AES256' }, bkt: {} }, + {}, + 'after', + ); }); }); @@ -424,8 +532,11 @@ describe('ensure MPU use good SSE', () => { await helpers.s3.createBucket({ Bucket: mpuKmsBkt }); await helpers.s3.putBucketEncryption({ Bucket: mpuKmsBkt, - ServerSideEncryptionConfiguration: - helpers.hydrateSSEConfig({ algo: 'aws:kms', masterKeyId: kmsKeympuKmsBkt }) }); + ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ + algo: 'aws:kms', + masterKeyId: kmsKeympuKmsBkt, + }), + }); }); after(async () => { @@ -435,7 +546,9 @@ describe('ensure MPU use good SSE', () => { it('mpu upload part should fail with sse header', async () => { const key = 'mpuKeyBadUpload'; const mpu = await helpers.s3.createMultipartUpload({ - Bucket: mpuKmsBkt, Key: key }); + Bucket: mpuKmsBkt, + Key: key, + }); const res = await promisify(makeRequest)({ method: 'PUT', hostname: helpers.s3.config.endpoint.hostname, @@ -465,21 +578,33 @@ describe('ensure MPU use good SSE', () => { const key = 'mpuKey'; const mpuKms = (await helpers.createKmsKey(log)).masterKeyArn; const mpu = await helpers.s3.createMultipartUpload({ - Bucket: mpuKmsBkt, Key: key, ServerSideEncryption: 'aws:kms', SSEKMSKeyId: mpuKms }); + Bucket: mpuKmsBkt, + Key: key, + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: mpuKms, + }); assert.strictEqual(mpu.ServerSideEncryption, 'aws:kms'); assert.strictEqual(mpu.SSEKMSKeyId, helpers.getKey(mpuKms)); - const part1 = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: mpuKmsBkt, - Body: 'Scality', - Key: key, - PartNumber: 1, - }, mpu, 'aws:kms', 'after'); + const part1 = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: mpuKmsBkt, + Body: 'Scality', + Key: key, + PartNumber: 1, + }, + mpu, + 'aws:kms', + 'after', + ); await scenarios.tests.mpuComplete( { UploadId: mpu.UploadId, Bucket: mpuKmsBkt, Key: key }, { existingParts: [], newParts: [part1] }, - mpu, 'aws:kms', 'after'); + mpu, + 'aws:kms', + 'after', + ); const assertion = { Bucket: mpuKmsBkt, @@ -492,16 +617,12 @@ describe('ensure MPU use good SSE', () => { }; const bktForAssert = { bktConf: { algo: 'aws:kms', masterKeyId: true }, - bkt: { kmsKey: kmsKeympuKmsBkt, - kmsKeyInfo: { masterKeyId: kmsKeympuKmsBkt, masterKeyArn: kmsKeympuKmsBkt } }, + bkt: { + kmsKey: kmsKeympuKmsBkt, + kmsKeyInfo: { masterKeyId: kmsKeympuKmsBkt, masterKeyArn: kmsKeympuKmsBkt }, + }, }; - await scenarios.assertObjectSSE( - assertion, - objForAssert, - bktForAssert, - {}, - 'after', - ); + await scenarios.assertObjectSSE(assertion, objForAssert, bktForAssert, {}, 'after'); }); }); @@ -533,10 +654,11 @@ describe('KMS error', () => { */ const expectedLocalKms = { code: 'KMS.AccessDeniedException', - msg: () => new RegExp( - 'The ciphertext refers to a customer master key that does not exist, ' + - 'does not exist in this region, or you are not allowed to access\\.' - ), + msg: () => + new RegExp( + 'The ciphertext refers to a customer master key that does not exist, ' + + 'does not exist in this region, or you are not allowed to access\\.', + ), }; if (helpers.config.backends.kms === 'kmip') { expected = expectedKMIP; @@ -567,8 +689,7 @@ describe('KMS error', () => { Body: body, }); - mpuPlaintext = await helpers.s3.createMultipartUpload( - helpers.putObjParams(Bucket, 'mpuPlaintext', {}, null)); + mpuPlaintext = await helpers.s3.createMultipartUpload(helpers.putObjParams(Bucket, 'mpuPlaintext', {}, null)); ({ masterKeyId, masterKeyArn } = await helpers.createKmsKey(log)); @@ -578,7 +699,8 @@ describe('KMS error', () => { assert.strictEqual(obj.Body.toString(), body); mpuEncrypted = await helpers.s3.createMultipartUpload( - helpers.putObjParams(Bucket, 'mpuEncrypted', sseConfig, masterKeyArn)); + helpers.putObjParams(Bucket, 'mpuEncrypted', sseConfig, masterKeyArn), + ); // make key unavailable await helpers.destroyKmsKey(masterKeyArn, log); @@ -589,70 +711,89 @@ describe('KMS error', () => { if (masterKeyArn) { try { await helpers.destroyKmsKey(masterKeyArn, log); - } catch (e) { void e; } + } catch (e) { + void e; + } [masterKeyArn, masterKeyId] = [null, null]; } }); const testCases = [ { - action: 'putObject', kmsAction: 'Encrypt', - fct: async ({ masterKeyArn }) => - helpers.putEncryptedObject(Bucket, 'fail', sseConfig, masterKeyArn, body), + action: 'putObject', + kmsAction: 'Encrypt', + fct: async ({ masterKeyArn }) => helpers.putEncryptedObject(Bucket, 'fail', sseConfig, masterKeyArn, body), }, { - action: 'getObject', kmsAction: 'Decrypt', + action: 'getObject', + kmsAction: 'Decrypt', fct: async () => helpers.s3.getObject({ Bucket, Key }), }, { - action: 'copyObject', detail: ' when getting from source', kmsAction: 'Decrypt', - fct: async () => - helpers.s3.copyObject({ Bucket, Key: 'copy', CopySource: `${Bucket}/${Key}` }), + action: 'copyObject', + detail: ' when getting from source', + kmsAction: 'Decrypt', + fct: async () => helpers.s3.copyObject({ Bucket, Key: 'copy', CopySource: `${Bucket}/${Key}` }), }, { - action: 'copyObject', detail: ' when putting to destination', kmsAction: 'Encrypt', - fct: async ({ masterKeyArn }) => helpers.s3.copyObject({ - Bucket, - Key: 'copyencrypted', - CopySource: `${Bucket}/plaintext`, - ServerSideEncryption: 'aws:kms', - SSEKMSKeyId: masterKeyArn, - }), + action: 'copyObject', + detail: ' when putting to destination', + kmsAction: 'Encrypt', + fct: async ({ masterKeyArn }) => + helpers.s3.copyObject({ + Bucket, + Key: 'copyencrypted', + CopySource: `${Bucket}/plaintext`, + ServerSideEncryption: 'aws:kms', + SSEKMSKeyId: masterKeyArn, + }), }, { - action: 'createMPU', kmsAction: 'Encrypt', - fct: async ({ masterKeyArn }) => helpers.s3.createMultipartUpload( - helpers.putObjParams(Bucket, 'mpuKeyEncryptedFail', sseConfig, masterKeyArn)) , + action: 'createMPU', + kmsAction: 'Encrypt', + fct: async ({ masterKeyArn }) => + helpers.s3.createMultipartUpload( + helpers.putObjParams(Bucket, 'mpuKeyEncryptedFail', sseConfig, masterKeyArn), + ), }, { - action: 'mpu uploadPartCopy', detail: ' when getting from source', kmsAction: 'Decrypt', - fct: async ({ mpuPlaintext }) => helpers.s3.uploadPartCopy({ - UploadId: mpuPlaintext.UploadId, - Bucket, - Key: 'mpuPlaintext', - PartNumber: 1, - CopySource: `${Bucket}/${Key}`, - }), + action: 'mpu uploadPartCopy', + detail: ' when getting from source', + kmsAction: 'Decrypt', + fct: async ({ mpuPlaintext }) => + helpers.s3.uploadPartCopy({ + UploadId: mpuPlaintext.UploadId, + Bucket, + Key: 'mpuPlaintext', + PartNumber: 1, + CopySource: `${Bucket}/${Key}`, + }), }, { - action: 'mpu uploadPart', detail: ' when putting to destination', kmsAction: 'Encrypt', - fct: async ({ mpuEncrypted }) => helpers.s3.uploadPart({ - UploadId: mpuEncrypted.UploadId, - Bucket, - Key: 'mpuEncrypted', - PartNumber: 1, - Body: body, - }), + action: 'mpu uploadPart', + detail: ' when putting to destination', + kmsAction: 'Encrypt', + fct: async ({ mpuEncrypted }) => + helpers.s3.uploadPart({ + UploadId: mpuEncrypted.UploadId, + Bucket, + Key: 'mpuEncrypted', + PartNumber: 1, + Body: body, + }), }, { - action: 'mpu uploadPartCopy', detail: ' when putting to destination', kmsAction: 'Encrypt', - fct: async ({ mpuEncrypted }) => helpers.s3.uploadPartCopy({ - UploadId: mpuEncrypted.UploadId, - Bucket, - Key: 'mpuEncrypted', - PartNumber: 1, - CopySource: `${Bucket}/plaintext`, - }), + action: 'mpu uploadPartCopy', + detail: ' when putting to destination', + kmsAction: 'Encrypt', + fct: async ({ mpuEncrypted }) => + helpers.s3.uploadPartCopy({ + UploadId: mpuEncrypted.UploadId, + Bucket, + Key: 'mpuEncrypted', + PartNumber: 1, + CopySource: `${Bucket}/plaintext`, + }), }, ]; diff --git a/tests/functional/sse-kms-migration/beforeMigration.js b/tests/functional/sse-kms-migration/beforeMigration.js index 1a7a024761..135603e119 100644 --- a/tests/functional/sse-kms-migration/beforeMigration.js +++ b/tests/functional/sse-kms-migration/beforeMigration.js @@ -12,7 +12,11 @@ const scenarios = require('./scenarios'); // copy part of aws-node-sdk/test/object/encryptionHeaders.js and add more tests // Fix for before migration run to not add a prefix -Object.defineProperty(kms, 'arnPrefix', { get() { return ''; } }); +Object.defineProperty(kms, 'arnPrefix', { + get() { + return ''; + }, +}); describe('SSE KMS before migration', () => { /** Bucket to test CopyObject from and to */ @@ -36,9 +40,7 @@ describe('SSE KMS before migration', () => { if (bktConf.algo && bktConf.masterKeyId) { const key = crypto.randomBytes(32).toString('hex'); bkt.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; - bkt.kmsKey = bktConf.arnPrefix - ? bkt.kmsKeyInfo.masterKeyArn - : bkt.kmsKeyInfo.masterKeyId; + bkt.kmsKey = bktConf.arnPrefix ? bkt.kmsKeyInfo.masterKeyArn : bkt.kmsKeyInfo.masterKeyId; } await helpers.s3.createBucket({ Bucket: bkt.name }); await helpers.s3.createBucket({ Bucket: bkt.vname }); @@ -47,38 +49,44 @@ describe('SSE KMS before migration', () => { await helpers.s3.putBucketEncryption({ Bucket: bkt.name, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); await helpers.s3.putBucketEncryption({ Bucket: bkt.vname, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: bktConf.algo, masterKeyId: bkt.kmsKey }), + algo: bktConf.algo, + masterKeyId: bkt.kmsKey, + }), }); } // Put an object for each SSE conf in each bucket - await Promise.all(scenarios.testCases.map(async objConf => { - const obj = { - name: `for-copy-enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(for-copy-enc-obj-${objConf.name})`, - }; - bkt.objs[objConf.name] = obj; - if (objConf.algo && objConf.masterKeyId) { - const key = crypto.randomBytes(32).toString('hex'); - obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } - return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); - })); + await Promise.all( + scenarios.testCases.map(async objConf => { + const obj = { + name: `for-copy-enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(for-copy-enc-obj-${objConf.name})`, + }; + bkt.objs[objConf.name] = obj; + if (objConf.algo && objConf.masterKeyId) { + const key = crypto.randomBytes(32).toString('hex'); + obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + return await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); + }), + ); }; before(async () => { - console.log('Run before migration', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run before migration', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = ((await helpers.s3.listBuckets({})).Buckets || []).map(b => b.Name); console.log('List buckets:', allBuckets); await promisify(metadata.setup.bind(metadata))(); @@ -96,337 +104,457 @@ describe('SSE KMS before migration', () => { await Promise.all(scenarios.testCases.map(async bktConf => this.initBucket(bktConf))); }); - scenarios.testCases.forEach(bktConf => describe(`bucket enc-bkt-${bktConf.name}`, () => { - let bkt = bkts[bktConf.name]; - - before(() => { - bkt = bkts[bktConf.name]; - }); + scenarios.testCases.forEach(bktConf => + describe(`bucket enc-bkt-${bktConf.name}`, () => { + let bkt = bkts[bktConf.name]; - if (bktConf.deleteSSE) { - beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); - } - - if (!bktConf.algo) { - it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', - async () => await scenarios.tests.getBucketSSEError(bkt.name)); + before(() => { + bkt = bkts[bktConf.name]; + }); - if (!bktConf.deleteSSE) { - it('should have non mandatory SSE in bucket MD as test init put an object with AES256', - async () => await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'before')); + if (bktConf.deleteSSE) { + beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); } - } else { - it('GetBucketEncryption should return SSE with arnPrefix to key', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, bktConf.algo, - bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'before')); - } - scenarios.testCasesObj.forEach(objConf => it(`should have pre uploaded object with SSE ${objConf.name}`, - async () => scenarios.tests.getPreUploadedObject(bkt.name, - { objConf, obj: bkt.objs[objConf.name] }, { bktConf, bkt }))); + if (!bktConf.algo) { + it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', async () => + await scenarios.tests.getBucketSSEError(bkt.name)); - scenarios.testCasesObj.forEach(objConf => describe(`object enc-obj-${objConf.name}`, () => { - const obj = { - name: `enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(enc-obj-${objConf.name})`, - }; - /** to be used as source of copy */ - let objForCopy; - - before(async () => { - if (objConf.algo && objConf.masterKeyId) { - const key = crypto.randomBytes(32).toString('hex'); - obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; + if (!bktConf.deleteSSE) { + it('should have non mandatory SSE in bucket MD as test init put an object with AES256', async () => + await scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'before')); } - objForCopy = bkt.objs[objConf.name]; - }); - - it(`should PutObject ${obj.name} overriding bucket SSE`, - async () => scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt })); - - // CopyObject scenarios - [ - { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, - { name: `${obj.name} into same bucket with object SSE config` }, - { name: `from encrypted source into ${obj.name} with object SSE config` }, - ].forEach(({ name, forceBktSSE }, index) => - it(`should CopyObject ${name}`, async () => - await scenarios.tests.copyObjectAndSSE( - { copyBkt, objForCopy, copyObj }, - { objConf, obj }, - { bktConf, bkt }, - { index, forceBktSSE }, + } else { + it('GetBucketEncryption should return SSE with arnPrefix to key', async () => + await scenarios.tests.getBucketSSE( + bkt.name, + log, + bktConf.algo, + bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'before', - ))); - - // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket - // Fixed at the same time as migration, some scenario can pass only in newer version above migration - const optionalSkip = objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) - ? it.skip - : it; - optionalSkip('should encrypt MPU and put 2 encrypted parts', async () => { - const mpuKey = `${obj.name}-mpu`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; - const newParts = []; - for (const [index, body] of partsBody.entries()) { - const part = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: body, - Key: mpuKey, - PartNumber: index + 1, - }, mpu, objConf.algo || bktConf.algo, 'before'); - newParts.push(part); - } - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'before'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `${obj.body}-MPU1${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); - }); - - optionalSkip('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const part1 = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpu, objConf.algo || bktConf.algo, 'before'); - const part2 = await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: 2, - }, mpu, objConf.algo || bktConf.algo, 'before'); - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts: [part1, part2] }, - mpu, objConf.algo || bktConf.algo, 'before'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `BODY(copy)${obj.body}-MPU2`, - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); - }); - - optionalSkip('should encrypt MPU and copy an encrypted range parts from encrypted bucket', async () => { - const mpuKey = `${obj.name}-mpucopyrange`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - const newParts = []; - for (const [index, range] of sourceRanges.entries()) { - const part = await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpu, objConf.algo || bktConf.algo, 'before'); - newParts.push(part); - } - - await scenarios.tests.mpuComplete( - { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts: [], newParts }, - mpu, objConf.algo || bktConf.algo, 'before'); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: 'copyBODY', - }; - await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); - }); - - optionalSkip('should prepare empty encrypted MPU without completion', async () => { - const mpuKey = `${obj.name}-migration-mpu-empty`; - await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - }); - - optionalSkip('should prepare encrypted MPU and put 2 encrypted parts without completion', async () => { - const mpuKey = `${obj.name}-migration-mpu`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; - for (const [index, body] of partsBody.entries()) { - await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: body, - Key: mpuKey, - PartNumber: index + 1, - }, mpu, objConf.algo || bktConf.algo, 'before'); - } - }); - - optionalSkip('should prepare encrypted MPU and copy an encrypted parts ' + - 'from encrypted bucket without completion', async () => { - const mpuKey = `${obj.name}-migration-mpucopy`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpu, objConf.algo || bktConf.algo, 'before'); - await scenarios.tests.mpuUploadPart({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: 2, - }, mpu, objConf.algo || bktConf.algo, 'before'); - }); - - optionalSkip('should prepare encrypted MPU and copy an encrypted range parts ' + - 'from encrypted bucket without completion', async () => { - const mpuKey = `${obj.name}-migration-mpucopyrange`; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey)); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - for (const [index, range] of sourceRanges.entries()) { - await scenarios.tests.mpuUploadPartCopy({ - UploadId: mpu.UploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpu, objConf.algo || bktConf.algo, 'before'); - } - }); + )); + } - it(`should PutObject versioned with SSE ${obj.name}`, async () => { - // ensure versioned bucket is empty - await helpers.bucketUtil.empty(bkt.vname); - let { Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname }) || { Versions: [] }; - assert.strictEqual(Versions.length, 0); - - const bodyBase = `BODY(${obj.name})-base`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); - const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyBase }, - { objConf, obj }, { bktConf, bkt }); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 1); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Enabled' }, - }); - - const bodyV1 = `BODY(${obj.name})-v1`; - const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); - const bodyV2 = `BODY(${obj.name})-v2`; - const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); - assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - await helpers.s3.putBucketVersioning({ Bucket: bkt.vname, - VersioningConfiguration: { Status: 'Suspended' }, - }); - - // should be fine after version suspension - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); // v2 - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: 'null', Body: bodyBase }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, { objConf, obj }, { bktConf, bkt }); - await scenarios.assertObjectSSE( - { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, { objConf, obj }, { bktConf, bkt }); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); - - // put a new null version - const bodyFinal = `BODY(${obj.name})-final`; - await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }); // null - await scenarios.assertObjectSSE( - { ...baseAssertion, Body: bodyFinal }, { objConf, obj }, { bktConf, bkt }, 'null'); - ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); - assert.strictEqual(Versions.length, 3); + scenarios.testCasesObj.forEach(objConf => + it(`should have pre uploaded object with SSE ${objConf.name}`, async () => + scenarios.tests.getPreUploadedObject( + bkt.name, + { objConf, obj: bkt.objs[objConf.name] }, + { bktConf, bkt }, + )), + ); + + scenarios.testCasesObj.forEach(objConf => + describe(`object enc-obj-${objConf.name}`, () => { + const obj = { + name: `enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(enc-obj-${objConf.name})`, + }; + /** to be used as source of copy */ + let objForCopy; + + before(async () => { + if (objConf.algo && objConf.masterKeyId) { + const key = crypto.randomBytes(32).toString('hex'); + obj.kmsKeyInfo = { masterKeyId: key, masterKeyArn: `${kms.arnPrefix}${key}` }; + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + objForCopy = bkt.objs[objConf.name]; + }); + + it(`should PutObject ${obj.name} overriding bucket SSE`, async () => + scenarios.tests.putObjectOverrideSSE({ objConf, obj }, { bktConf, bkt })); + + // CopyObject scenarios + [ + { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, + { name: `${obj.name} into same bucket with object SSE config` }, + { name: `from encrypted source into ${obj.name} with object SSE config` }, + ].forEach(({ name, forceBktSSE }, index) => + it(`should CopyObject ${name}`, async () => + await scenarios.tests.copyObjectAndSSE( + { copyBkt, objForCopy, copyObj }, + { objConf, obj }, + { bktConf, bkt }, + { index, forceBktSSE }, + 'before', + )), + ); + + // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket + // Fixed at the same time as migration, some scenario can pass only in newer version above migration + const optionalSkip = + objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) ? it.skip : it; + optionalSkip('should encrypt MPU and put 2 encrypted parts', async () => { + const mpuKey = `${obj.name}-mpu`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; + const newParts = []; + for (const [index, body] of partsBody.entries()) { + const part = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: index + 1, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + newParts.push(part); + } + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `${obj.body}-MPU1${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); + }); + + optionalSkip('should encrypt MPU and copy an encrypted parts from encrypted bucket', async () => { + const mpuKey = `${obj.name}-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const part1 = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const part2 = await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: 2, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts: [part1, part2] }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `BODY(copy)${obj.body}-MPU2`, + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); + }); + + optionalSkip( + 'should encrypt MPU and copy an encrypted range parts from encrypted bucket', + async () => { + const mpuKey = `${obj.name}-mpucopyrange`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + const newParts = []; + for (const [index, range] of sourceRanges.entries()) { + const part = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + newParts.push(part); + } + + await scenarios.tests.mpuComplete( + { UploadId: mpu.UploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts: [], newParts }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: 'copyBODY', + }; + await scenarios.assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }); + }, + ); + + optionalSkip('should prepare empty encrypted MPU without completion', async () => { + const mpuKey = `${obj.name}-migration-mpu-empty`; + await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + }); + + optionalSkip( + 'should prepare encrypted MPU and put 2 encrypted parts without completion', + async () => { + const mpuKey = `${obj.name}-migration-mpu`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + const partsBody = [`${obj.body}-MPU1`, `${obj.body}-MPU2`]; + for (const [index, body] of partsBody.entries()) { + await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: index + 1, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + } + }, + ); + + optionalSkip( + 'should prepare encrypted MPU and copy an encrypted parts ' + + 'from encrypted bucket without completion', + async () => { + const mpuKey = `${obj.name}-migration-mpucopy`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + await scenarios.tests.mpuUploadPart( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: 2, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + }, + ); + + optionalSkip( + 'should prepare encrypted MPU and copy an encrypted range parts ' + + 'from encrypted bucket without completion', + async () => { + const mpuKey = `${obj.name}-migration-mpucopyrange`; + const mpu = await helpers.s3.createMultipartUpload( + helpers.putObjParams(bkt.name, mpuKey, objConf, obj.kmsKey), + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + for (const [index, range] of sourceRanges.entries()) { + await scenarios.tests.mpuUploadPartCopy( + { + UploadId: mpu.UploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpu, + objConf.algo || bktConf.algo, + 'before', + ); + } + }, + ); + + it(`should PutObject versioned with SSE ${obj.name}`, async () => { + // ensure versioned bucket is empty + await helpers.bucketUtil.empty(bkt.vname); + let { Versions } = (await helpers.s3.listObjectVersions({ Bucket: bkt.vname })) || { + Versions: [], + }; + assert.strictEqual(Versions.length, 0); + + const bodyBase = `BODY(${obj.name})-base`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyBase); + const baseAssertion = { Bucket: bkt.vname, Key: obj.name }; + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 1); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Enabled' }, + }); + + const bodyV1 = `BODY(${obj.name})-v1`; + const v1 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV1); + const bodyV2 = `BODY(${obj.name})-v2`; + const v2 = await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyV2); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + const current = await helpers.s3.headObject({ Bucket: bkt.vname, Key: obj.name }); + assert.strictEqual(current.VersionId, v2.VersionId); // ensure versioning as expected + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + await helpers.s3.putBucketVersioning({ + Bucket: bkt.vname, + VersioningConfiguration: { Status: 'Suspended' }, + }); + + // should be fine after version suspension + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); // v2 + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: 'null', Body: bodyBase }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v1.VersionId, Body: bodyV1 }, + { objConf, obj }, + { bktConf, bkt }, + ); + await scenarios.assertObjectSSE( + { ...baseAssertion, VersionId: v2.VersionId, Body: bodyV2 }, + { objConf, obj }, + { bktConf, bkt }, + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + + // put a new null version + const bodyFinal = `BODY(${obj.name})-final`; + await helpers.putEncryptedObject(bkt.vname, obj.name, objConf, obj.kmsKey, bodyFinal); + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + ); // null + await scenarios.assertObjectSSE( + { ...baseAssertion, Body: bodyFinal }, + { objConf, obj }, + { bktConf, bkt }, + 'null', + ); + ({ Versions } = await helpers.s3.listObjectVersions({ Bucket: bkt.vname })); + assert.strictEqual(Versions.length, 3); + }); + }), + ); + }), + ); + + it( + 'should prepare encrypted MPU and copy parts from ' + 'every buckets and objects matrice without completion', + async () => { + await helpers.s3.putBucketEncryption({ + Bucket: mpuCopyBkt, + // AES256 because input key is broken for now + ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }), }); - })); - })); - - it('should prepare encrypted MPU and copy parts from ' + - 'every buckets and objects matrice without completion', async () => { - await helpers.s3.putBucketEncryption({ - Bucket: mpuCopyBkt, - // AES256 because input key is broken for now - ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ algo: 'AES256' }), - }); - const mpuKey = 'mpucopy'; - const mpu = await helpers.s3.createMultipartUpload( - helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); - const copyPartArg = { - UploadId: mpu.UploadId, - Bucket: mpuCopyBkt, - Key: mpuKey, - }; - // For each test Case bucket and object copy a part - const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { - const bkt = bkts[bktConf.name]; - - return acc.concat(scenarios.testCasesObj.map(async (objConf, objIdx) => { - const obj = bkt.objs[objConf.name]; - - const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; - const res = await helpers.s3.uploadPartCopy({ - ...copyPartArg, - PartNumber: partNumber, - CopySource: `${bkt.name}/${obj.name}`, - }); - - return { partNumber, body: obj.body, res: res.CopyPartResult }; - })); - }, []); - - await Promise.all(uploadPromises); - }); + const mpuKey = 'mpucopy'; + const mpu = await helpers.s3.createMultipartUpload(helpers.putObjParams(mpuCopyBkt, mpuKey, {}, null)); + const copyPartArg = { + UploadId: mpu.UploadId, + Bucket: mpuCopyBkt, + Key: mpuKey, + }; + // For each test Case bucket and object copy a part + const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { + const bkt = bkts[bktConf.name]; + + return acc.concat( + scenarios.testCasesObj.map(async (objConf, objIdx) => { + const obj = bkt.objs[objConf.name]; + + const partNumber = bktIdx * scenarios.testCasesObj.length + objIdx + 1; + const res = await helpers.s3.uploadPartCopy({ + ...copyPartArg, + PartNumber: partNumber, + CopySource: `${bkt.name}/${obj.name}`, + }); + + return { partNumber, body: obj.body, res: res.CopyPartResult }; + }), + ); + }, []); + + await Promise.all(uploadPromises); + }, + ); }); diff --git a/tests/functional/sse-kms-migration/cleanup.js b/tests/functional/sse-kms-migration/cleanup.js index d1e5cb32e2..5be7b64b87 100644 --- a/tests/functional/sse-kms-migration/cleanup.js +++ b/tests/functional/sse-kms-migration/cleanup.js @@ -16,8 +16,10 @@ describe('SSE KMS Cleanup', () => { const mpuCopyBkt = 'enc-bkt-mpu-copy'; it('Empty and delete buckets for SSE KMS Migration', async () => { - console.log('Run cleanup', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run cleanup', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = ((await helpers.s3.listBuckets()).Buckets || []).map(b => b.Name); console.log('List buckets:', allBuckets); @@ -26,10 +28,14 @@ describe('SSE KMS Cleanup', () => { try { await cleanup(copyBkt); await cleanup(mpuCopyBkt); - await Promise.all(scenarios.testCases.map(async bktConf => { - await cleanup(`enc-bkt-${bktConf.name}`); - return await cleanup(`versioned-enc-bkt-${bktConf.name}`); - })); - } catch (e) { void e; } + await Promise.all( + scenarios.testCases.map(async bktConf => { + await cleanup(`enc-bkt-${bktConf.name}`); + return await cleanup(`versioned-enc-bkt-${bktConf.name}`); + }), + ); + } catch (e) { + void e; + } }); }); diff --git a/tests/functional/sse-kms-migration/configs/aws.json b/tests/functional/sse-kms-migration/configs/aws.json index c593f310f5..913e031c99 100644 --- a/tests/functional/sse-kms-migration/configs/aws.json +++ b/tests/functional/sse-kms-migration/configs/aws.json @@ -1,9 +1,8 @@ { - "kmsAWS": { "noAwsArn": true, "providerName": "local", - "region": "us-east-1", + "region": "us-east-1", "endpoint": "http://0:8080", "ak": "456", "sk": "123" diff --git a/tests/functional/sse-kms-migration/configs/base.json b/tests/functional/sse-kms-migration/configs/base.json index 841aa341c7..9db4bea239 100644 --- a/tests/functional/sse-kms-migration/configs/base.json +++ b/tests/functional/sse-kms-migration/configs/base.json @@ -12,28 +12,33 @@ "127.0.0.2": "us-east-1", "s3.amazonaws.com": "us-east-1" }, - "websiteEndpoints": ["s3-website-us-east-1.amazonaws.com", - "s3-website.us-east-2.amazonaws.com", - "s3-website-us-west-1.amazonaws.com", - "s3-website-us-west-2.amazonaws.com", - "s3-website.ap-south-1.amazonaws.com", - "s3-website.ap-northeast-2.amazonaws.com", - "s3-website-ap-southeast-1.amazonaws.com", - "s3-website-ap-southeast-2.amazonaws.com", - "s3-website-ap-northeast-1.amazonaws.com", - "s3-website.eu-central-1.amazonaws.com", - "s3-website-eu-west-1.amazonaws.com", - "s3-website-sa-east-1.amazonaws.com", - "s3-website.localhost", - "s3-website.scality.test"], - "replicationEndpoints": [{ - "site": "zenko", - "servers": ["127.0.0.1:8000"], - "default": true - }, { - "site": "us-east-2", - "type": "aws_s3" - }], + "websiteEndpoints": [ + "s3-website-us-east-1.amazonaws.com", + "s3-website.us-east-2.amazonaws.com", + "s3-website-us-west-1.amazonaws.com", + "s3-website-us-west-2.amazonaws.com", + "s3-website.ap-south-1.amazonaws.com", + "s3-website.ap-northeast-2.amazonaws.com", + "s3-website-ap-southeast-1.amazonaws.com", + "s3-website-ap-southeast-2.amazonaws.com", + "s3-website-ap-northeast-1.amazonaws.com", + "s3-website.eu-central-1.amazonaws.com", + "s3-website-eu-west-1.amazonaws.com", + "s3-website-sa-east-1.amazonaws.com", + "s3-website.localhost", + "s3-website.scality.test" + ], + "replicationEndpoints": [ + { + "site": "zenko", + "servers": ["127.0.0.1:8000"], + "default": true + }, + { + "site": "us-east-2", + "type": "aws_s3" + } + ], "cdmi": { "host": "localhost", "port": 81, diff --git a/tests/functional/sse-kms-migration/configs/kmip-cluster.json b/tests/functional/sse-kms-migration/configs/kmip-cluster.json index 70ffcaf1b5..0723ab7260 100644 --- a/tests/functional/sse-kms-migration/configs/kmip-cluster.json +++ b/tests/functional/sse-kms-migration/configs/kmip-cluster.json @@ -4,7 +4,7 @@ "client": { "compoundCreateActivate": false }, - "transport": [ + "transport": [ { "pipelineDepth": 8, "tls": { diff --git a/tests/functional/sse-kms-migration/helpers.js b/tests/functional/sse-kms-migration/helpers.js index 456cade69d..c2e24bc1e2 100644 --- a/tests/functional/sse-kms-migration/helpers.js +++ b/tests/functional/sse-kms-migration/helpers.js @@ -1,5 +1,6 @@ const getConfig = require('../aws-node-sdk/test/support/config'); -const { S3Client, +const { + S3Client, CreateBucketCommand, DeleteBucketCommand, PutBucketEncryptionCommand, @@ -52,7 +53,7 @@ const httpsAgent = new HttpsAgent({ keepAliveMsecs: 30000, maxSockets: 50, maxFreeSockets: 10, - timeout: 120000, + timeout: 120000, }); const s3config = { @@ -82,22 +83,24 @@ const s3 = { putBucketEncryption: params => wrap(() => s3Client.send(new PutBucketEncryptionCommand(params))), getBucketEncryption: params => wrap(() => s3Client.send(new GetBucketEncryptionCommand(params))), putObject: params => wrap(() => s3Client.send(new PutObjectCommand(params))), - getObject: params => wrap(async () => { - const response = await s3Client.send(new GetObjectCommand(params)); - const body = await response.Body.transformToString(); - return { ...response, Body: body }; - }), + getObject: params => + wrap(async () => { + const response = await s3Client.send(new GetObjectCommand(params)); + const body = await response.Body.transformToString(); + return { ...response, Body: body }; + }), listBuckets: params => wrap(() => s3Client.send(new ListBucketsCommand(params || {}))), copyObject: params => wrap(() => s3Client.send(new CopyObjectCommand(params))), - listObjectVersions: params => wrap(async () => { - const response = await s3Client.send(new ListObjectVersionsCommand(params)); - return { - ...response, - Versions: response.Versions || [], - DeleteMarkers: response.DeleteMarkers || [], - CommonPrefixes: response.CommonPrefixes || [] - }; - }), + listObjectVersions: params => + wrap(async () => { + const response = await s3Client.send(new ListObjectVersionsCommand(params)); + return { + ...response, + Versions: response.Versions || [], + DeleteMarkers: response.DeleteMarkers || [], + CommonPrefixes: response.CommonPrefixes || [], + }; + }), headObject: params => wrap(() => s3Client.send(new HeadObjectCommand(params))), createMultipartUpload: params => wrap(() => s3Client.send(new CreateMultipartUploadCommand(params))), uploadPart: params => wrap(() => s3Client.send(new UploadPartCommand(params))), @@ -105,14 +108,15 @@ const s3 = { completeMultipartUpload: params => wrap(() => s3Client.send(new CompleteMultipartUploadCommand(params))), putBucketVersioning: params => wrap(() => s3Client.send(new PutBucketVersioningCommand(params))), headBucket: params => wrap(() => s3Client.send(new HeadBucketCommand(params))), - listMultipartUploads: params => wrap(async () => { - const response = await s3Client.send(new ListMultipartUploadsCommand(params)); - return { - ...response, - Uploads: response.Uploads || [], - CommonPrefixes: response.CommonPrefixes || [] - }; - }), + listMultipartUploads: params => + wrap(async () => { + const response = await s3Client.send(new ListMultipartUploadsCommand(params)); + return { + ...response, + Uploads: response.Uploads || [], + CommonPrefixes: response.CommonPrefixes || [], + }; + }), listParts: params => wrap(() => s3Client.send(new ListPartsCommand(params))), _compat: bucketUtil.s3, config: { @@ -129,13 +133,18 @@ const s3 = { function hydrateSSEConfig({ algo: SSEAlgorithm, masterKeyId: KMSMasterKeyID }) { // Stringify and parse to strip undefined values - return JSON.parse(JSON.stringify({ Rules: [{ - ApplyServerSideEncryptionByDefault: { - SSEAlgorithm, - KMSMasterKeyID, - }, - }], - })); + return JSON.parse( + JSON.stringify({ + Rules: [ + { + ApplyServerSideEncryptionByDefault: { + SSEAlgorithm, + KMSMasterKeyID, + }, + }, + ], + }), + ); } function putObjParams(Bucket, Key, sseConfig, kmsKeyId) { diff --git a/tests/functional/sse-kms-migration/load.js b/tests/functional/sse-kms-migration/load.js index 33773fa899..786eb841e7 100644 --- a/tests/functional/sse-kms-migration/load.js +++ b/tests/functional/sse-kms-migration/load.js @@ -37,7 +37,6 @@ async function spawnTcpdump(port, packetCount) { detached: true, stdio: ['ignore', 'pipe', 'pipe'], // ignored stdin shell: false, // no need as it's detached - }, ); let stderr = ''; @@ -51,7 +50,8 @@ async function spawnTcpdump(port, packetCount) { spawnTimeout = setTimeout(() => { if (child.exitCode !== null || child.signalCode !== null) { const err = `countPacketsByIp.sh stopped after spawn with code ${ - child.exitCode} and signal ${child.signalCode}.\nStderr: ${stderr}`; + child.exitCode + } and signal ${child.signalCode}.\nStderr: ${stderr}`; reject(new Error(err)); } else { resolve(child); @@ -70,9 +70,7 @@ async function spawnTcpdump(port, packetCount) { if (spawnTimeout) { clearTimeout(spawnTimeout); } - reject(new Error( - `tcpdump script closed with code ${code} and signal ${signal}.\nStderr: ${stderr}` - )); + reject(new Error(`tcpdump script closed with code ${code} and signal ${signal}.\nStderr: ${stderr}`)); } }); }); @@ -89,8 +87,9 @@ async function stopTcpdump(tcpdump) { }); } -describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER - } objs each in ${BUCKET_NUMBER} bkts (${TOTAL_OBJECTS} objs)`, () => { +describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${ + OBJECT_NUMBER +} objs each in ${BUCKET_NUMBER} bkts (${TOTAL_OBJECTS} objs)`, () => { let buckets = []; let tcpdumpProcess; let stdout; @@ -107,18 +106,23 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER await helpers.s3.putBucketEncryption({ Bucket, ServerSideEncryptionConfiguration: helpers.hydrateSSEConfig({ - algo: 'aws:kms', masterKeyId: masterKeyArn }), + algo: 'aws:kms', + masterKeyId: masterKeyArn, + }), }); return { Bucket, masterKeyArn }; - })); + }), + ); }); after(async () => { - await Promise.all(buckets.map(async ({ Bucket, masterKeyArn }) => { - await helpers.cleanup(Bucket); - return helpers.destroyKmsKey(masterKeyArn, log); - })); + await Promise.all( + buckets.map(async ({ Bucket, masterKeyArn }) => { + await helpers.cleanup(Bucket); + return helpers.destroyKmsKey(masterKeyArn, log); + }), + ); await promisify(kms.client.stop.bind(kms.client))(); }); @@ -147,7 +151,7 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER const [count, ip] = line.trim().split(' '); return { count: +count, ip }; }), - }) + }), ); }); }); @@ -175,30 +179,37 @@ describe(`KMS load (kmip cluster ${KMS_NODES} nodes): ${OBJECT_NUMBER const repartitionCount = repartition.map(({ count }) => count); assert.strictEqual(code, 0, `tcpdump script closed with code ${code} and signal ${signal}`); assert(repartition.length === KMS_NODES, `Expected ${KMS_NODES} IPs but got ${repartition.length}`); - assert(repartitionCount.every(count => - count >= EXPECTED_MIN && count <= EXPECTED_MAX), + assert( + repartitionCount.every(count => count >= EXPECTED_MIN && count <= EXPECTED_MAX), `Repartition counts should be around ${TOTAL_OBJECTS_PER_NODE} ` + - `(±${APPROX}, min: ${EXPECTED_MIN}, max: ${EXPECTED_MAX}) but got ${repartitionCount}`); + `(±${APPROX}, min: ${EXPECTED_MIN}, max: ${EXPECTED_MAX}) but got ${repartitionCount}`, + ); } it(`should encrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => { - await (Promise.all( - buckets.map(async ({ Bucket }) => Promise.all( - // Send little more request in case a packet is missed. - new Array(OBJECT_NUMBER + 1).fill(0).map(async (_, i) => - helpers.s3.putObject({ Bucket, Key: `obj-${i}`, Body: `body-${i}` })) - )) - )); + await Promise.all( + buckets.map(async ({ Bucket }) => + Promise.all( + // Send little more request in case a packet is missed. + new Array(OBJECT_NUMBER + 1) + .fill(0) + .map(async (_, i) => helpers.s3.putObject({ Bucket, Key: `obj-${i}`, Body: `body-${i}` })), + ), + ), + ); await assertRepartition(closePromise); }); it(`should decrypt ${TOTAL_OBJECTS} times in parallel, ~${TOTAL_OBJECTS_PER_NODE} per node`, async () => { await Promise.all( - buckets.map(async ({ Bucket }) => Promise.all( - // Send little more request in case a packet is missed. - new Array(OBJECT_NUMBER + 1).fill(0).map(async (_, i) => - helpers.s3.getObject({ Bucket, Key: `obj-${i}` })) - )) + buckets.map(async ({ Bucket }) => + Promise.all( + // Send little more request in case a packet is missed. + new Array(OBJECT_NUMBER + 1) + .fill(0) + .map(async (_, i) => helpers.s3.getObject({ Bucket, Key: `obj-${i}` })), + ), + ), ); await assertRepartition(closePromise); }); diff --git a/tests/functional/sse-kms-migration/migration.js b/tests/functional/sse-kms-migration/migration.js index f34748c223..7b85e32e57 100644 --- a/tests/functional/sse-kms-migration/migration.js +++ b/tests/functional/sse-kms-migration/migration.js @@ -25,16 +25,17 @@ async function assertObjectSSE( const sseMD = await helpers.getObjectMDSSE(Bucket, Key); const head = await helpers.s3.headObject({ Bucket, Key, VersionId }); const sseMDMigrated = await helpers.getObjectMDSSE(Bucket, Key); - const expectedKey = `${sseMD.SSEKMSKeyId && isScalityKmsArn(sseMD.SSEKMSKeyId) - ? '' : arnPrefix}${sseMD.SSEKMSKeyId}`; + const expectedKey = `${ + sseMD.SSEKMSKeyId && isScalityKmsArn(sseMD.SSEKMSKeyId) ? '' : arnPrefix + }${sseMD.SSEKMSKeyId}`; if (!put && sseMD.SSEKMSKeyId) { assert.doesNotMatch(sseMD.SSEKMSKeyId, SCAL_KMS_ARN_REG); } // obj precedence over bkt - assert.strictEqual(head.ServerSideEncryption, (objConf.algo || bktConf.algo)); - headers && assert.strictEqual(headers.ServerSideEncryption, (objConf.algo || bktConf.algo)); + assert.strictEqual(head.ServerSideEncryption, objConf.algo || bktConf.algo); + headers && assert.strictEqual(headers.ServerSideEncryption, objConf.algo || bktConf.algo); if (sseMDMigrated.SSEKMSKeyId) { // on metadata verify the full key with arn prefix @@ -82,12 +83,10 @@ describe('SSE KMS migration', () => { bkts[bktConf.name] = bkt; if (bktConf.algo && bktConf.masterKeyId) { bkt.kmsKeyInfo = await helpers.createKmsKey(log); - bkt.kmsKey = bktConf.arnPrefix - ? bkt.kmsKeyInfo.masterKeyArn - : bkt.kmsKeyInfo.masterKeyId; + bkt.kmsKey = bktConf.arnPrefix ? bkt.kmsKeyInfo.masterKeyArn : bkt.kmsKeyInfo.masterKeyId; } - await helpers.s3.headBucket(({ Bucket: bkt.name })); - await helpers.s3.headBucket(({ Bucket: bkt.vname })); + await helpers.s3.headBucket({ Bucket: bkt.name }); + await helpers.s3.headBucket({ Bucket: bkt.vname }); if (bktConf.algo) { const bktSSE = await helpers.getBucketSSE(bkt.name); assert.strictEqual(bktSSE.SSEAlgorithm, bktConf.algo); @@ -103,38 +102,40 @@ describe('SSE KMS migration', () => { } // Check object SSE using MD api, not S3 to avoid triggering migration - await Promise.all(scenarios.testCases.map(async objConf => { - const obj = { - name: `for-copy-enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(for-copy-enc-obj-${objConf.name})`, - }; - bkt.objs[objConf.name] = obj; - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } - const objSSE = await helpers.getObjectMDSSE(bkt.name, obj.name); - assert.strictEqual(objSSE.ServerSideEncryption, objConf.algo || bktConf.algo || ''); - assert.doesNotMatch(objSSE.SSEKMSKeyId, SCAL_KMS_ARN_REG); - return undefined; - })); + await Promise.all( + scenarios.testCases.map(async objConf => { + const obj = { + name: `for-copy-enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(for-copy-enc-obj-${objConf.name})`, + }; + bkt.objs[objConf.name] = obj; + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + const objSSE = await helpers.getObjectMDSSE(bkt.name, obj.name); + assert.strictEqual(objSSE.ServerSideEncryption, objConf.algo || bktConf.algo || ''); + assert.doesNotMatch(objSSE.SSEKMSKeyId, SCAL_KMS_ARN_REG); + return undefined; + }), + ); }; before('setup', async () => { - console.log('Run migration', - { profile: helpers.credsProfile, accessKeyId: helpers.s3.config.credentials.accessKeyId }); + console.log('Run migration', { + profile: helpers.credsProfile, + accessKeyId: helpers.s3.config.credentials.accessKeyId, + }); const allBuckets = (await helpers.s3.listBuckets()).Buckets.map(b => b.Name); console.log('List buckets:', allBuckets); await helpers.MD.setup(); await helpers.s3.headBucket({ Bucket: copyBkt }); - await helpers.s3.headBucket(({ Bucket: mpuCopyBkt })); + await helpers.s3.headBucket({ Bucket: mpuCopyBkt }); const copySSE = await helpers.s3.getBucketEncryption({ Bucket: copyBkt }); - const { SSEAlgorithm, KMSMasterKeyID } = copySSE - .ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault; + const { SSEAlgorithm, KMSMasterKeyID } = + copySSE.ServerSideEncryptionConfiguration.Rules[0].ApplyServerSideEncryptionByDefault; assert.strictEqual(SSEAlgorithm, 'aws:kms'); assert.doesNotMatch(KMSMasterKeyID, SCAL_KMS_ARN_REG); @@ -146,241 +147,289 @@ describe('SSE KMS migration', () => { await helpers.cleanup(copyBkt); await helpers.cleanup(mpuCopyBkt); // Clean every bucket - await Promise.all(Object.values(bkts).map(async bkt => { - await helpers.cleanup(bkt.name); - return await helpers.cleanup(bkt.vname); - })); + await Promise.all( + Object.values(bkts).map(async bkt => { + await helpers.cleanup(bkt.name); + return await helpers.cleanup(bkt.vname); + }), + ); }); - scenarios.testCases.forEach(bktConf => describe(`bucket enc-bkt-${bktConf.name}`, () => { - let bkt = bkts[bktConf.name]; - - before(() => { - bkt = bkts[bktConf.name]; - }); - - if (bktConf.deleteSSE) { - beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); - } + scenarios.testCases.forEach(bktConf => + describe(`bucket enc-bkt-${bktConf.name}`, () => { + let bkt = bkts[bktConf.name]; - if (!bktConf.algo) { - it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', - async () => await scenarios.tests.getBucketSSEError(bkt.name)); + before(() => { + bkt = bkts[bktConf.name]; + }); - if (!bktConf.deleteSSE) { - it('should have non mandatory SSE in bucket MD as test init put an object with AES256', - async () => scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'migration')); + if (bktConf.deleteSSE) { + beforeEach(async () => scenarios.deleteBucketSSEBeforeEach(bkt.name, log)); } - } else { - it('ensure old SSE KMS key setup', - async () => await scenarios.tests.getBucketSSE(bkt.name, log, bktConf.algo, - bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, 'migration')); - } - scenarios.testCasesObj.forEach(objConf => it(`should have pre uploaded object with SSE ${objConf.name}`, - async () => { - const obj = bkt.objs[objConf.name]; - // use MD here to avoid triggering a migration - const sseMD = await helpers.getObjectMDSSE(bkt.name, obj.name); - if (sseMD.SSEKMSKeyId) { - assert.doesNotMatch(sseMD.SSEKMSKeyId, SCAL_KMS_ARN_REG); - } - })); - - scenarios.testCasesObj.forEach(objConf => describe(`object enc-obj-${objConf.name}`, () => { - const obj = { - name: `enc-obj-${objConf.name}`, - kmsKeyInfo: null, - kmsKey: null, - body: `BODY(enc-obj-${objConf.name})`, - }; - /** to be used as source of copy */ - let objForCopy; - - before(async () => { - if (objConf.algo && objConf.masterKeyId) { - obj.kmsKeyInfo = await helpers.createKmsKey(log); - obj.kmsKey = objConf.arnPrefix - ? obj.kmsKeyInfo.masterKeyArn - : obj.kmsKeyInfo.masterKeyId; - } - objForCopy = bkt.objs[objConf.name]; - }); + if (!bktConf.algo) { + it('GetBucketEncryption should return ServerSideEncryptionConfigurationNotFoundError', async () => + await scenarios.tests.getBucketSSEError(bkt.name)); - const mpus = {}; - before('retrieve MPUS', async () => { - const listed = await helpers.s3.listMultipartUploads({ Bucket: bkt.name }); - assert.strictEqual(listed.IsTruncated, false, 'Too much MPUs, need to loop on pagination'); - for (const mpu of listed.Uploads) { - mpus[mpu.Key] = mpu.UploadId; + if (!bktConf.deleteSSE) { + it('should have non mandatory SSE in bucket MD as test init put an object with AES256', async () => + scenarios.tests.getBucketNonMandatorySSE(bkt.name, log, 'migration')); } - }); - - it(`should PutObject ${obj.name} overriding bucket SSE`, async () => { - await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); - const assertion = { - Bucket: bkt.name, - Key: obj.name, - Body: obj.body, - }; - await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, { put: true }); - }); - - // CopyObject scenarios - [ - { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, - { name: `${obj.name} into same bucket with object SSE config` }, - { name: `from encrypted source into ${obj.name} with object SSE config` }, - ].forEach(({ name, forceBktSSE }, index) => - it(`should CopyObject ${name}`, async () => - await scenarios.tests.copyObjectAndSSE( - { copyBkt, objForCopy, copyObj }, - { objConf, obj }, - { bktConf, bkt }, - { index, forceBktSSE, assertObjectSSEFct: assertObjectSSE }, - ))); - - // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket - // Fixed at the same time as migration, some scenario can pass only in newer version above migration - const optionalSkip = objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) - ? it.skip - : it; - - // completed MPU should behave like regular objects - [ - { name: '', keySuffix: '', body: `${obj.body}-MPU1${obj.body}-MPU2` }, - { name: 'that has copy', keySuffix: 'copy', body: `BODY(copy)${obj.body}-MPU2` }, - { name: 'that has byte range copy', keySuffix: 'copyrange', body: 'copyBODY' }, - ].forEach(({ name, keySuffix, body }) => - optionalSkip(`should migrate completed MPU ${name}`, async () => { - const mpuKey = `${obj.name}-mpu${keySuffix}`; - const assertion = { Bucket: bkt.name, Key: mpuKey, Body: body }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - })); - - async function prepareMPUTest(mpuKey, expectedExistingParts) { - const uploadId = mpus[mpuKey]; - assert(uploadId, 'Missing MPU, it should have been prepared before'); - const MPUBucketName = `${mpuBucketPrefix}${bkt.name}`; - const longMPUIdentifier = `overview${splitter}${mpuKey}${splitter}${uploadId}`; - const mpuOverviewMDSSE = await helpers.getObjectMDSSE(MPUBucketName, longMPUIdentifier); - - const existingParts = await helpers.s3.listParts({ - Bucket: bkt.name, Key: mpuKey, UploadId: uploadId }); - const partCount = (existingParts.Parts || []).length || 0; - assert.strictEqual(existingParts.IsTruncated, false, 'Too much parts, need to loop on pagination'); - assert.strictEqual(partCount, expectedExistingParts); - return { mpuKey, uploadId, mpuOverviewMDSSE, partCount, existingParts: existingParts.Parts || [] }; + } else { + it('ensure old SSE KMS key setup', async () => + await scenarios.tests.getBucketSSE( + bkt.name, + log, + bktConf.algo, + bktConf.masterKeyId ? bkt.kmsKeyInfo.masterKeyArn : null, + 'migration', + )); } - // ongoing MPU with regular uploadPart - [ - { - name: 'empty', - keySuffix: '-empty', - existingPartsCount: 0, - partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], - body: `${obj.body}-MPU1${obj.body}-MPU2`, - }, - { - name: 'with 2 parts', - keySuffix: '', - existingPartsCount: 2, - partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], - body: `${obj.body}-MPU1${obj.body}-MPU2`.repeat(2), - }, - ].forEach(({ name, keySuffix, existingPartsCount, partsBody, body }) => - optionalSkip(`should finish ongoing encrypted MPU ${name} by adding 2 parts`, async () => { - const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = - await prepareMPUTest(`${obj.name}-migration-mpu${keySuffix}`, existingPartsCount); - const newParts = []; - for (const [index, body] of partsBody.entries()) { - const part = await scenarios.tests.mpuUploadPart({ - UploadId: uploadId, + scenarios.testCasesObj.forEach(objConf => + it(`should have pre uploaded object with SSE ${objConf.name}`, async () => { + const obj = bkt.objs[objConf.name]; + // use MD here to avoid triggering a migration + const sseMD = await helpers.getObjectMDSSE(bkt.name, obj.name); + if (sseMD.SSEKMSKeyId) { + assert.doesNotMatch(sseMD.SSEKMSKeyId, SCAL_KMS_ARN_REG); + } + }), + ); + + scenarios.testCasesObj.forEach(objConf => + describe(`object enc-obj-${objConf.name}`, () => { + const obj = { + name: `enc-obj-${objConf.name}`, + kmsKeyInfo: null, + kmsKey: null, + body: `BODY(enc-obj-${objConf.name})`, + }; + /** to be used as source of copy */ + let objForCopy; + + before(async () => { + if (objConf.algo && objConf.masterKeyId) { + obj.kmsKeyInfo = await helpers.createKmsKey(log); + obj.kmsKey = objConf.arnPrefix ? obj.kmsKeyInfo.masterKeyArn : obj.kmsKeyInfo.masterKeyId; + } + objForCopy = bkt.objs[objConf.name]; + }); + + const mpus = {}; + before('retrieve MPUS', async () => { + const listed = await helpers.s3.listMultipartUploads({ Bucket: bkt.name }); + assert.strictEqual(listed.IsTruncated, false, 'Too much MPUs, need to loop on pagination'); + for (const mpu of listed.Uploads) { + mpus[mpu.Key] = mpu.UploadId; + } + }); + + it(`should PutObject ${obj.name} overriding bucket SSE`, async () => { + await helpers.putEncryptedObject(bkt.name, obj.name, objConf, obj.kmsKey, obj.body); + const assertion = { + Bucket: bkt.name, + Key: obj.name, + Body: obj.body, + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, { put: true }); + }); + + // CopyObject scenarios + [ + { name: `${obj.name} into encrypted destination bucket`, forceBktSSE: true }, + { name: `${obj.name} into same bucket with object SSE config` }, + { name: `from encrypted source into ${obj.name} with object SSE config` }, + ].forEach(({ name, forceBktSSE }, index) => + it(`should CopyObject ${name}`, async () => + await scenarios.tests.copyObjectAndSSE( + { copyBkt, objForCopy, copyObj }, + { objConf, obj }, + { bktConf, bkt }, + { index, forceBktSSE, assertObjectSSEFct: assertObjectSSE }, + )), + ); + + // S3C-9996 The SSE was bugged with MPU, where the completion takes only the masterKeyId from bucket + // Fixed at the same time as migration, some scenario can pass only in newer version above migration + const optionalSkip = + objConf.algo || bktConf.masterKeyId || (!bktConf.algo && !bktConf.deleteSSE) ? it.skip : it; + + // completed MPU should behave like regular objects + [ + { name: '', keySuffix: '', body: `${obj.body}-MPU1${obj.body}-MPU2` }, + { name: 'that has copy', keySuffix: 'copy', body: `BODY(copy)${obj.body}-MPU2` }, + { name: 'that has byte range copy', keySuffix: 'copyrange', body: 'copyBODY' }, + ].forEach(({ name, keySuffix, body }) => + optionalSkip(`should migrate completed MPU ${name}`, async () => { + const mpuKey = `${obj.name}-mpu${keySuffix}`; + const assertion = { Bucket: bkt.name, Key: mpuKey, Body: body }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }), + ); + + async function prepareMPUTest(mpuKey, expectedExistingParts) { + const uploadId = mpus[mpuKey]; + assert(uploadId, 'Missing MPU, it should have been prepared before'); + const MPUBucketName = `${mpuBucketPrefix}${bkt.name}`; + const longMPUIdentifier = `overview${splitter}${mpuKey}${splitter}${uploadId}`; + const mpuOverviewMDSSE = await helpers.getObjectMDSSE(MPUBucketName, longMPUIdentifier); + + const existingParts = await helpers.s3.listParts({ Bucket: bkt.name, - Body: body, Key: mpuKey, - PartNumber: partCount + index + 1, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - newParts.push(part); + UploadId: uploadId, + }); + const partCount = (existingParts.Parts || []).length || 0; + assert.strictEqual( + existingParts.IsTruncated, + false, + 'Too much parts, need to loop on pagination', + ); + assert.strictEqual(partCount, expectedExistingParts); + return { + mpuKey, + uploadId, + mpuOverviewMDSSE, + partCount, + existingParts: existingParts.Parts || [], + }; } - await scenarios.tests.mpuComplete( - { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts, newParts }, - mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: body, - }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - })); - - optionalSkip('should finish ongoing encrypted MPU with 2 parts by copy and upload part', async () => { - const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = - await prepareMPUTest(`${obj.name}-migration-mpucopy`, 2); - const part1 = await scenarios.tests.mpuUploadPartCopy({ - UploadId: uploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: partCount + 1, - CopySource: `${copyBkt}/${copyObj}`, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const part2 = await scenarios.tests.mpuUploadPart({ - UploadId: uploadId, - Bucket: bkt.name, - Body: `${obj.body}-MPU2`, - Key: mpuKey, - PartNumber: partCount + 2, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - await scenarios.tests.mpuComplete( - { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts, newParts: [part1, part2] }, - mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: `BODY(copy)${obj.body}-MPU2`.repeat(2), - }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - }); - - optionalSkip('should finish ongoing encrypted MPU with 2 parts by 2 copy byte range', async () => { - const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = - await prepareMPUTest(`${obj.name}-migration-mpucopyrange`, 2); - // source body is "BODY(copy)" - // [copy, BODY] - const sourceRanges = ['bytes=5-8', 'bytes=0-3']; - const newParts = []; - for (const [index, range] of sourceRanges.entries()) { - const part = await scenarios.tests.mpuUploadPartCopy({ - UploadId: uploadId, - Bucket: bkt.name, - Key: mpuKey, - PartNumber: partCount + index + 1, - CopySource: `${copyBkt}/${copyObj}`, - CopySourceRange: range, - }, mpuOverviewMDSSE, objConf.algo || bktConf.algo); - newParts.push(part); - } - await scenarios.tests.mpuComplete( - { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, - { existingParts, newParts }, - mpuOverviewMDSSE, objConf.algo || bktConf.algo); - const assertion = { - Bucket: bkt.name, - Key: mpuKey, - Body: 'copyBODY'.repeat(2), - }; - await assertObjectSSE( - assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); - }); - })); - })); + // ongoing MPU with regular uploadPart + [ + { + name: 'empty', + keySuffix: '-empty', + existingPartsCount: 0, + partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], + body: `${obj.body}-MPU1${obj.body}-MPU2`, + }, + { + name: 'with 2 parts', + keySuffix: '', + existingPartsCount: 2, + partsBody: [`${obj.body}-MPU1`, `${obj.body}-MPU2`], + body: `${obj.body}-MPU1${obj.body}-MPU2`.repeat(2), + }, + ].forEach(({ name, keySuffix, existingPartsCount, partsBody, body }) => + optionalSkip(`should finish ongoing encrypted MPU ${name} by adding 2 parts`, async () => { + const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = + await prepareMPUTest(`${obj.name}-migration-mpu${keySuffix}`, existingPartsCount); + const newParts = []; + for (const [index, body] of partsBody.entries()) { + const part = await scenarios.tests.mpuUploadPart( + { + UploadId: uploadId, + Bucket: bkt.name, + Body: body, + Key: mpuKey, + PartNumber: partCount + index + 1, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + newParts.push(part); + } + await scenarios.tests.mpuComplete( + { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts, newParts }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: body, + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }), + ); + + optionalSkip( + 'should finish ongoing encrypted MPU with 2 parts by copy and upload part', + async () => { + const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = + await prepareMPUTest(`${obj.name}-migration-mpucopy`, 2); + const part1 = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: uploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: partCount + 1, + CopySource: `${copyBkt}/${copyObj}`, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const part2 = await scenarios.tests.mpuUploadPart( + { + UploadId: uploadId, + Bucket: bkt.name, + Body: `${obj.body}-MPU2`, + Key: mpuKey, + PartNumber: partCount + 2, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + await scenarios.tests.mpuComplete( + { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts, newParts: [part1, part2] }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: `BODY(copy)${obj.body}-MPU2`.repeat(2), + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }, + ); + + optionalSkip('should finish ongoing encrypted MPU with 2 parts by 2 copy byte range', async () => { + const { mpuKey, uploadId, mpuOverviewMDSSE, existingParts, partCount } = await prepareMPUTest( + `${obj.name}-migration-mpucopyrange`, + 2, + ); + // source body is "BODY(copy)" + // [copy, BODY] + const sourceRanges = ['bytes=5-8', 'bytes=0-3']; + const newParts = []; + for (const [index, range] of sourceRanges.entries()) { + const part = await scenarios.tests.mpuUploadPartCopy( + { + UploadId: uploadId, + Bucket: bkt.name, + Key: mpuKey, + PartNumber: partCount + index + 1, + CopySource: `${copyBkt}/${copyObj}`, + CopySourceRange: range, + }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + newParts.push(part); + } + + await scenarios.tests.mpuComplete( + { UploadId: uploadId, Bucket: bkt.name, Key: mpuKey }, + { existingParts, newParts }, + mpuOverviewMDSSE, + objConf.algo || bktConf.algo, + ); + const assertion = { + Bucket: bkt.name, + Key: mpuKey, + Body: 'copyBODY'.repeat(2), + }; + await assertObjectSSE(assertion, { objConf, obj }, { bktConf, bkt }, fileArnPrefix); + }); + }), + ); + }), + ); it('should finish ongoing encrypted MPU by copy parts from all bkt and objects matrice', async () => { const mpuKey = 'mpucopy'; @@ -403,18 +452,20 @@ describe('SSE KMS migration', () => { const uploadPromises = scenarios.testCases.reduce((acc, bktConf, bktIdx) => { const bkt = bkts[bktConf.name]; - return acc.concat(scenarios.testCasesObj.map(async (objConf, objIdx) => { - const obj = bkt.objs[objConf.name]; + return acc.concat( + scenarios.testCasesObj.map(async (objConf, objIdx) => { + const obj = bkt.objs[objConf.name]; - const partNumber = partCount + bktIdx * scenarios.testCasesObj.length + objIdx + 1; - const res = await helpers.s3.uploadPartCopy({ - ...copyPartArg, - PartNumber: partNumber, - CopySource: `${bkt.name}/${obj.name}`, - }); + const partNumber = partCount + bktIdx * scenarios.testCasesObj.length + objIdx + 1; + const res = await helpers.s3.uploadPartCopy({ + ...copyPartArg, + PartNumber: partNumber, + CopySource: `${bkt.name}/${obj.name}`, + }); - return { partNumber, body: obj.body, res: res.CopyPartResult }; - })); + return { partNumber, body: obj.body, res: res.CopyPartResult }; + }), + ); }, []); const parts = await Promise.all(uploadPromises); @@ -436,6 +487,10 @@ describe('SSE KMS migration', () => { Body: parts.reduce((acc, part) => `${acc}${part.body}`, '').repeat(2), }; await assertObjectSSE( - assertion, { objConf: {}, obj: {} }, { bktConf: { algo: 'AES256' }, bkt: {} }, fileArnPrefix); + assertion, + { objConf: {}, obj: {} }, + { bktConf: { algo: 'AES256' }, bkt: {} }, + fileArnPrefix, + ); }); }); diff --git a/tests/functional/sse-kms-migration/scenarios.js b/tests/functional/sse-kms-migration/scenarios.js index 008a5c7491..8d344256e2 100644 --- a/tests/functional/sse-kms-migration/scenarios.js +++ b/tests/functional/sse-kms-migration/scenarios.js @@ -52,10 +52,10 @@ async function assertObjectSSE( const sseMD = await helpers.getObjectMDSSE(Bucket, Key); const arnPrefixReg = new RegExp(`^${arnPrefix}`); - const expectedAlgo = (objConf.algo || bktConf.algo) || - (testCase === 'after' && helpers.config.globalEncryptionEnabled && !bktConf.deleteSSE - ? 'AES256' - : undefined); + const expectedAlgo = + objConf.algo || + bktConf.algo || + (testCase === 'after' && helpers.config.globalEncryptionEnabled && !bktConf.deleteSSE ? 'AES256' : undefined); // obj precedence over bkt assert.strictEqual(head.ServerSideEncryption, expectedAlgo); @@ -209,8 +209,7 @@ async function copyObjectAndSSE( const { SSEAlgorithm, KMSMasterKeyID } = await helpers.getBucketSSE(copyBkt); assert.strictEqual(headers.ServerSideEncryption, SSEAlgorithm); testCase !== 'before' && assert.strictEqual(headers.SSEKMSKeyId, KMSMasterKeyID); - const keyArn = `${KMSMasterKeyID && isScalityKmsArn(KMSMasterKeyID) - ? '' : kms.arnPrefix}${KMSMasterKeyID}`; + const keyArn = `${KMSMasterKeyID && isScalityKmsArn(KMSMasterKeyID) ? '' : kms.arnPrefix}${KMSMasterKeyID}`; const kmsKeyInfo = { masterKeyId: getKeyIdFromArn(keyArn), masterKeyArn: keyArn, @@ -263,7 +262,9 @@ async function mpuUploadPart({ UploadId, Bucket, Key, Body, PartNumber }, mpuOve // before has no headers to assert async function mpuUploadPartCopy( { UploadId, Bucket, Key, PartNumber, CopySource, CopySourceRange }, - mpuOverviewMDSSE, algo, testCase + mpuOverviewMDSSE, + algo, + testCase, ) { const part = await helpers.s3.uploadPartCopy({ UploadId, @@ -284,23 +285,23 @@ async function mpuComplete({ UploadId, Bucket, Key }, { existingParts, newParts assert(eTag !== undefined, `Could not find ETag in part: ${JSON.stringify(part)}`); return eTag; }; - + const allParts = [ - ...existingParts.map(part => ({ - PartNumber: part.PartNumber, - ETag: extractETag(part) + ...existingParts.map(part => ({ + PartNumber: part.PartNumber, + ETag: extractETag(part), })), - ...newParts.map((part, idx) => ({ - PartNumber: existingParts.length + idx + 1, - ETag: extractETag(part) + ...newParts.map((part, idx) => ({ + PartNumber: existingParts.length + idx + 1, + ETag: extractETag(part), })), - ]; + ]; const complete = await helpers.s3.completeMultipartUpload({ UploadId, Bucket, Key, MultipartUpload: { - Parts: allParts, + Parts: allParts, }, }); testCase !== 'before' && assertMPUSSEHeaders(complete, mpuOverviewMDSSE, algo); diff --git a/tests/locationConfig/locationConfigLegacy.json b/tests/locationConfig/locationConfigLegacy.json index 73dfed5179..c078a81ace 100644 --- a/tests/locationConfig/locationConfigLegacy.json +++ b/tests/locationConfig/locationConfigLegacy.json @@ -1,4 +1,3 @@ - { "legacy": { "type": "mem", diff --git a/tests/multipleBackend/backendHealthcheckResponse.js b/tests/multipleBackend/backendHealthcheckResponse.js index d24c4e3d70..b5b85fe253 100644 --- a/tests/multipleBackend/backendHealthcheckResponse.js +++ b/tests/multipleBackend/backendHealthcheckResponse.js @@ -2,8 +2,7 @@ const assert = require('assert'); const DummyRequestLogger = require('../unit/helpers').DummyRequestLogger; -const clientCheck - = require('../../lib/utilities/healthcheckHandler').clientCheck; +const clientCheck = require('../../lib/utilities/healthcheckHandler').clientCheck; const { config } = require('../../lib/Config'); const { getAzureClient, @@ -11,10 +10,7 @@ const { getAzureContainerName, } = require('../functional/aws-node-sdk/test/multipleBackend/utils'); -const { - LOCATION_NAME_DMF, - LOCATION_NAME_CRR, -} = require('../constants'); +const { LOCATION_NAME_DMF, LOCATION_NAME_CRR } = require('../constants'); const log = new DummyRequestLogger(); const locConstraints = Object.keys(config.locationConstraints); @@ -23,55 +19,59 @@ const azureClient = getAzureClient(); describe('Healthcheck response', function describeHealthcheck() { this.timeout(60000); - it('should return result for every location constraint in ' + - 'locationConfig and every external locations with flightCheckOnStartUp ' + - 'set to true', done => { - clientCheck(true, log, (err, results) => { - const resultKeys = Object.keys(results); - locConstraints.forEach(constraint => { - if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { - // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 - return; - } - assert(resultKeys.includes(constraint), `constraint: ${constraint} not in results: ${resultKeys}`); + it( + 'should return result for every location constraint in ' + + 'locationConfig and every external locations with flightCheckOnStartUp ' + + 'set to true', + done => { + clientCheck(true, log, (err, results) => { + const resultKeys = Object.keys(results); + locConstraints.forEach(constraint => { + if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { + // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 + return; + } + assert(resultKeys.includes(constraint), `constraint: ${constraint} not in results: ${resultKeys}`); + }); + done(); }); - done(); - }); - }); - it('should return no error with flightCheckOnStartUp set to false', - done => { + }, + ); + it('should return no error with flightCheckOnStartUp set to false', done => { clientCheck(false, log, err => { - assert.strictEqual(err, null, - `Expected success but got error ${err}`); + assert.strictEqual(err, null, `Expected success but got error ${err}`); done(); }); }); - it('should return result for every location constraint in ' + - 'locationConfig and at least one of every external locations with ' + - 'flightCheckOnStartUp set to false', done => { - clientCheck(false, log, (err, results) => { - assert.notStrictEqual(results.length, locConstraints.length); - locConstraints.forEach(constraint => { - if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { - // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 - return; - } - if (Object.keys(results).indexOf(constraint) === -1) { - const locationType = config - .locationConstraints[constraint].type; - assert(Object.keys(results).some(result => - config.locationConstraints[result].type - === locationType)); - } + it( + 'should return result for every location constraint in ' + + 'locationConfig and at least one of every external locations with ' + + 'flightCheckOnStartUp set to false', + done => { + clientCheck(false, log, (err, results) => { + assert.notStrictEqual(results.length, locConstraints.length); + locConstraints.forEach(constraint => { + if (constraint === LOCATION_NAME_DMF || constraint === LOCATION_NAME_CRR) { + // FIXME: location-dmf-v1 and location-crr-v1 are not in results, see CLDSRV-440 + return; + } + if (Object.keys(results).indexOf(constraint) === -1) { + const locationType = config.locationConstraints[constraint].type; + assert( + Object.keys(results).some( + result => config.locationConstraints[result].type === locationType, + ), + ); + } + }); + done(); }); - done(); - }); - }); + }, + ); // FIXME: does not pass, see CLDSRV-441 describe.skip('Azure container creation', () => { - const containerName = - getAzureContainerName(azureLocationNonExistContainer); + const containerName = getAzureContainerName(azureLocationNonExistContainer); beforeEach(async () => { await azureClient.getContainerClient(containerName).deleteIfExists(); @@ -81,44 +81,57 @@ describe('Healthcheck response', function describeHealthcheck() { await azureClient.getContainerClient(containerName).deleteIfExists(); }); - it('should create an azure location\'s container if it is missing ' + - 'and the check is a flightCheckOnStartUp', done => { - clientCheck(true, log, (err, results) => { - const azureLocationNonExistContainerError = - results[azureLocationNonExistContainer].error; - if (err) { - assert(err.is.InternalError, `got unexpected err in clientCheck: ${err}`); - assert(azureLocationNonExistContainerError.startsWith( - 'The specified container is being deleted.')); - return done(); - } - return azureClient.getContainerClient(containerName).getProperties( - azureResult => { - assert.strictEqual(azureResult.metadata.name, containerName); - return done(); - }, err => { - assert.strictEqual(err, null, 'got unexpected err ' + - `heading azure container: ${err}`); + it( + "should create an azure location's container if it is missing " + 'and the check is a flightCheckOnStartUp', + done => { + clientCheck(true, log, (err, results) => { + const azureLocationNonExistContainerError = results[azureLocationNonExistContainer].error; + if (err) { + assert(err.is.InternalError, `got unexpected err in clientCheck: ${err}`); + assert( + azureLocationNonExistContainerError.startsWith('The specified container is being deleted.'), + ); return done(); - }); - }); - }); + } + return azureClient.getContainerClient(containerName).getProperties( + azureResult => { + assert.strictEqual(azureResult.metadata.name, containerName); + return done(); + }, + err => { + assert.strictEqual(err, null, 'got unexpected err ' + `heading azure container: ${err}`); + return done(); + }, + ); + }); + }, + ); - it('should not create an azure location\'s container even if it is ' + - 'missing if the check is not a flightCheckOnStartUp', done => { - clientCheck(false, log, err => { - assert.strictEqual(err, null, - `got unexpected err in clientCheck: ${err}`); - return azureClient.getContainerClient(containerName).getProperties().then( - () => { - assert(err, 'Expected err but did not find one'); - return done(); - }, err => { - assert.strictEqual(err.code, 'NotFound', - `got unexpected err code in clientCheck: ${err.code}`); - return done(); - }); - }); - }); + it( + "should not create an azure location's container even if it is " + + 'missing if the check is not a flightCheckOnStartUp', + done => { + clientCheck(false, log, err => { + assert.strictEqual(err, null, `got unexpected err in clientCheck: ${err}`); + return azureClient + .getContainerClient(containerName) + .getProperties() + .then( + () => { + assert(err, 'Expected err but did not find one'); + return done(); + }, + err => { + assert.strictEqual( + err.code, + 'NotFound', + `got unexpected err code in clientCheck: ${err.code}`, + ); + return done(); + }, + ); + }); + }, + ); }); }); diff --git a/tests/multipleBackend/multipartUpload.js b/tests/multipleBackend/multipartUpload.js index b5d537d73d..8edf6b6f8f 100644 --- a/tests/multipleBackend/multipartUpload.js +++ b/tests/multipleBackend/multipartUpload.js @@ -1,17 +1,12 @@ const assert = require('assert'); const async = require('async'); -const { S3Client, - HeadObjectCommand, - AbortMultipartUploadCommand, - ListPartsCommand } = require('@aws-sdk/client-s3'); +const { S3Client, HeadObjectCommand, AbortMultipartUploadCommand, ListPartsCommand } = require('@aws-sdk/client-s3'); const { parseString } = require('xml2js'); const { models } = require('arsenal'); const BucketInfo = models.BucketInfo; -const { getRealAwsConfig } = - require('../functional/aws-node-sdk/test/support/awsConfig'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = - require('../unit/helpers'); +const { getRealAwsConfig } = require('../functional/aws-node-sdk/test/support/awsConfig'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../unit/helpers'); const DummyRequest = require('../unit/DummyRequest'); const { config } = require('../../lib/Config'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; @@ -20,13 +15,11 @@ const { bucketPut } = require('../../lib/api/bucketPut'); const objectPut = require('../../lib/api/objectPut'); const objectGet = require('../../lib/api/objectGet'); const bucketPutVersioning = require('../../lib/api/bucketPutVersioning'); -const initiateMultipartUpload = - require('../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../lib/api/initiateMultipartUpload'); const multipartDelete = require('../../lib/api/multipartDelete'); const objectPutCopyPart = require('../../lib/api/objectPutCopyPart'); const objectPutPart = require('../../lib/api/objectPutPart'); -const completeMultipartUpload = - require('../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../lib/api/completeMultipartUpload'); const listParts = require('../../lib/api/listParts'); const listMultipartUploads = require('../../lib/api/listMultipartUploads'); const constants = require('../../constants'); @@ -47,8 +40,7 @@ const namespace = 'default'; const bucketName = 'bucketname'; const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; const awsBucket = config.locationConstraints[awsLocation].details.bucketName; -const awsMismatchBucket = config.locationConstraints[awsLocationMismatch] - .details.bucketName; +const awsMismatchBucket = config.locationConstraints[awsLocationMismatch].details.bucketName; const smallBody = Buffer.from('I am a body', 'utf8'); const bigBody = Buffer.alloc(10485760); const locMetaHeader = 'scal-location-constraint'; @@ -65,7 +57,8 @@ const bucketPutRequest = { const awsETag = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; const awsETagBigObj = 'f1c9645dbc14efddc7d8a322685f26eb'; const tagSet = 'key1=value1&key2=value2'; -const completeBody = '' + +const completeBody = + '' + '' + '1' + `"${awsETagBigObj}"` + @@ -83,29 +76,38 @@ const basicParams = { }; function getObjectGetRequest(objectKey) { - return Object.assign({ - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, basicParams); + return Object.assign( + { + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + basicParams, + ); } function getDeleteParams(objectKey, uploadId) { - return Object.assign({ - url: `/${objectKey}?uploadId=${uploadId}`, - query: { uploadId }, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - }, basicParams); + return Object.assign( + { + url: `/${objectKey}?uploadId=${uploadId}`, + query: { uploadId }, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + }, + basicParams, + ); } function getPartParams(objectKey, uploadId, partNumber) { - return Object.assign({ - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, - query: { partNumber, uploadId }, - }, basicParams); + return Object.assign( + { + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=${partNumber}&uploadId=${uploadId}`, + query: { partNumber, uploadId }, + }, + basicParams, + ); } function _getOverviewKey(objectKey, uploadId) { @@ -113,23 +115,29 @@ function _getOverviewKey(objectKey, uploadId) { } function getCompleteParams(objectKey, uploadId) { - return Object.assign({ - objectKey, - parsedHost: 's3.amazonaws.com', - headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: completeBody, - url: `/${objectKey}?uploadId=${uploadId}`, - query: { uploadId }, - }, basicParams); + return Object.assign( + { + objectKey, + parsedHost: 's3.amazonaws.com', + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: completeBody, + url: `/${objectKey}?uploadId=${uploadId}`, + query: { uploadId }, + }, + basicParams, + ); } function getListParams(objectKey, uploadId) { - return Object.assign({ - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?uploadId=${uploadId}`, - query: { uploadId }, - }, basicParams); + return Object.assign( + { + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${uploadId}`, + query: { uploadId }, + }, + basicParams, + ); } function getAwsParams(objectKey) { @@ -143,10 +151,8 @@ function getAwsParamsBucketNotMatch(objectKey) { function assertMpuInitResults(initResult, key, cb) { parseString(initResult, (err, json) => { assert.equal(err, null, `Error parsing mpu init results: ${err}`); - assert.strictEqual(json.InitiateMultipartUploadResult - .Bucket[0], bucketName); - assert.strictEqual(json.InitiateMultipartUploadResult - .Key[0], key); + assert.strictEqual(json.InitiateMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.InitiateMultipartUploadResult.Key[0], key); assert(json.InitiateMultipartUploadResult.UploadId[0]); cb(json.InitiateMultipartUploadResult.UploadId[0]); }); @@ -154,16 +160,13 @@ function assertMpuInitResults(initResult, key, cb) { function assertMpuCompleteResults(compResult, objectKey) { parseString(compResult, (err, json) => { - assert.equal(err, null, - `Error parsing mpu complete results: ${err}`); + assert.equal(err, null, `Error parsing mpu complete results: ${err}`); assert.strictEqual( json.CompleteMultipartUploadResult.Location[0], - `http://${bucketName}.s3.amazonaws.com/${objectKey}`); - assert.strictEqual( - json.CompleteMultipartUploadResult.Bucket[0], - bucketName); - assert.strictEqual( - json.CompleteMultipartUploadResult.Key[0], objectKey); + `http://${bucketName}.s3.amazonaws.com/${objectKey}`, + ); + assert.strictEqual(json.CompleteMultipartUploadResult.Bucket[0], bucketName); + assert.strictEqual(json.CompleteMultipartUploadResult.Key[0], objectKey); const MD = metadata.keyMaps.get(bucketName).get(objectKey); assert(MD); }); @@ -174,53 +177,42 @@ function assertListResults(listResult, testAttribute, uploadId, objectKey) { assert.equal(err, null, `Error parsing list part results: ${err}`); assert.strictEqual(json.ListPartsResult.Key[0], objectKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); // attributes to test specific to PartNumberMarker being set // in listParts if (testAttribute === 'partNumMarker') { - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); assert.strictEqual(json.ListPartsResult.Part.length, 1); assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], '1'); // data of second part put assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '2'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - `"${awsETag}"`); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], `"${awsETag}"`); assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '11'); } else { // common attributes to test if MaxParts set or // neither MaxParts nor PartNumberMarker set - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '1'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - `"${awsETagBigObj}"`); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '10485760'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], `"${awsETagBigObj}"`); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '10485760'); // attributes to test specific to MaxParts being set in listParts if (testAttribute === 'maxParts') { - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], - '1'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], '1'); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'true'); assert.strictEqual(json.ListPartsResult.Part.length, 1); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1'); } else { // attributes to test if neither MaxParts nor // PartNumberMarker set - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); assert.strictEqual(json.ListPartsResult.Part.length, 2); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1000'); - assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], - '2'); - assert.strictEqual(json.ListPartsResult.Part[1].ETag[0], - `"${awsETag}"`); + assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], '2'); + assert.strictEqual(json.ListPartsResult.Part[1].ETag[0], `"${awsETag}"`); assert.strictEqual(json.ListPartsResult.Part[1].Size[0], '11'); } } @@ -237,20 +229,20 @@ function _getZenkoObjectKey(objectKey) { function assertObjOnBackend(expectedBackend, objectKey, cb) { const zenkoObjectKey = _getZenkoObjectKey(objectKey); - return objectGet(authInfo, getObjectGetRequest(zenkoObjectKey), false, log, - async (err, result, metaHeaders) => { + return objectGet(authInfo, getObjectGetRequest(zenkoObjectKey), false, log, async (err, result, metaHeaders) => { assert.equal(err, null, `Error getting object on S3: ${err}`); assert.strictEqual(metaHeaders[`x-amz-meta-${locMetaHeader}`], expectedBackend); if (expectedBackend === awsLocation) { - return s3.send(new HeadObjectCommand({ Bucket: awsBucket, Key: objectKey })) - .then(result => { - assert.strictEqual(result.Metadata[locMetaHeader], awsLocation); - return cb(); - }).catch(err => { - assert.equal(err, null, 'Error on headObject call to AWS: ' + - `${err}`); - return cb(); - }); + return s3 + .send(new HeadObjectCommand({ Bucket: awsBucket, Key: objectKey })) + .then(result => { + assert.strictEqual(result.Metadata[locMetaHeader], awsLocation); + return cb(); + }) + .catch(err => { + assert.equal(err, null, 'Error on headObject call to AWS: ' + `${err}`); + return cb(); + }); } return process.nextTick(cb); }); @@ -275,14 +267,12 @@ function mpuSetup(location, key, cb) { bucketName, namespace, objectKey: key, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': location }, + headers: { host: `${bucketName}.s3.amazonaws.com`, 'x-amz-meta-scal-location-constraint': location }, url: `/${key}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.strictEqual(err, null, 'Error initiating MPU'); assertMpuInitResults(result, key, uploadId => { putParts(uploadId, key, () => { @@ -293,14 +283,17 @@ function mpuSetup(location, key, cb) { } function putObject(putBackend, objectKey, cb) { - const putParams = Object.assign({ - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': putBackend, + const putParams = Object.assign( + { + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': putBackend, + }, + url: '/', + objectKey, }, - url: '/', - objectKey, - }, basicParams); + basicParams, + ); const objectPutRequest = new DummyRequest(putParams, smallBody); return objectPut(authInfo, objectPutRequest, undefined, log, err => { assert.equal(err, null, `Error putting object to ${putBackend} ${err}`); @@ -313,22 +306,27 @@ function abortMPU(uploadId, awsParams, cb) { s3.send(new AbortMultipartUploadCommand(abortParams)) .then(() => { cb(); - }).catch(err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); + }) + .catch(err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); cb(); }); } function abortMultipleMpus(backendsInfo, callback) { - async.forEach(backendsInfo, (backend, cb) => { - const delParams = getDeleteParams(backend.key, backend.uploadId); - multipartDelete(authInfo, delParams, log, err => { - cb(err); - }); - }, err => { - assert.equal(err, null, `Error aborting MPU: ${err}`); - callback(); - }); + async.forEach( + backendsInfo, + (backend, cb) => { + const delParams = getDeleteParams(backend.key, backend.uploadId); + multipartDelete(authInfo, delParams, log, err => { + cb(err); + }); + }, + err => { + assert.equal(err, null, `Error aborting MPU: ${err}`); + callback(); + }, + ); } describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { @@ -351,15 +349,16 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { bucketName, namespace, objectKey, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${awsLocation}` }, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${awsLocation}`, + }, url: `/${objectKey}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.strictEqual(err, null, 'Error initiating MPU'); assertMpuInitResults(result, objectKey, uploadId => { abortMPU(uploadId, getAwsParams(objectKey), done); @@ -367,23 +366,22 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should initiate a multipart upload on AWS location with ' + - 'bucketMatch equals false', done => { + it('should initiate a multipart upload on AWS location with ' + 'bucketMatch equals false', done => { const objectKey = `key-${Date.now()}`; const initiateRequest = { bucketName, namespace, objectKey, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': - `${awsLocationMismatch}` }, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${awsLocationMismatch}`, + }, url: `/${objectKey}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.strictEqual(err, null, 'Error initiating MPU'); assertMpuInitResults(result, objectKey, uploadId => { abortMPU(uploadId, getAwsParamsBucketNotMatch(objectKey), done); @@ -398,7 +396,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { namespace, objectKey, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-meta-scal-location-constraint': `${awsLocation}`, 'x-amz-tagging': tagSet, }, @@ -407,8 +405,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - (err, result) => { + initiateMultipartUpload(authInfo, initiateRequest, log, (err, result) => { assert.ifError(err); assertMpuInitResults(result, objectKey, uploadId => { abortMPU(uploadId, getAwsParams(objectKey), done); @@ -428,8 +425,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should list the parts of a multipart upload on real AWS location ' + - 'with bucketMatch set to false', done => { + it('should list the parts of a multipart upload on real AWS location ' + 'with bucketMatch set to false', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocationMismatch, objectKey, uploadId => { const listParams = getListParams(objectKey, uploadId); @@ -441,8 +437,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should only return number of parts equal to specified maxParts', - function itF(done) { + it('should only return number of parts equal to specified maxParts', function itF(done) { this.timeout(90000); const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { @@ -476,7 +471,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { abortMPU(uploadId, getAwsParams(objectKey), () => { const listParams = getListParams(objectKey, uploadId); listParts(authInfo, listParams, log, err => { - const wantedDesc = 'Error returned from AWS: ' + + const wantedDesc = + 'Error returned from AWS: ' + 'The specified upload does not exist. The upload ID ' + 'may be invalid, or the upload may have been aborted' + ' or completed.'; @@ -494,37 +490,44 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const delParams = getDeleteParams(objectKey, uploadId); multipartDelete(authInfo, delParams, log, async err => { assert.equal(err, null, `Error aborting MPU: ${err}`); - s3.send(new ListPartsCommand({ - Bucket: awsBucket, - Key: objectKey, - UploadId: uploadId, - })).then(() => { - assert.fail('Expected an error listing parts of aborted MPU'); - }).catch(err => { - assert.strictEqual(err.name, 'NoSuchUpload'); - done(); - }); + s3.send( + new ListPartsCommand({ + Bucket: awsBucket, + Key: objectKey, + UploadId: uploadId, + }), + ) + .then(() => { + assert.fail('Expected an error listing parts of aborted MPU'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchUpload'); + done(); + }); }); }); }); - it('should abort a multipart upload on real AWS location with' + - 'bucketMatch set to false', done => { + it('should abort a multipart upload on real AWS location with' + 'bucketMatch set to false', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocationMismatch, objectKey, uploadId => { const delParams = getDeleteParams(objectKey, uploadId); multipartDelete(authInfo, delParams, log, async err => { assert.equal(err, null, `Error aborting MPU: ${err}`); - s3.send(new ListPartsCommand({ - Bucket: awsBucket, - Key: `${bucketName}/${objectKey}`, - UploadId: uploadId, - })).then(() => { - assert.fail('Expected an error listing parts of aborted MPU'); - }).catch(err => { - assert.strictEqual(err.name, 'NoSuchUpload'); - done(); - }); + s3.send( + new ListPartsCommand({ + Bucket: awsBucket, + Key: `${bucketName}/${objectKey}`, + UploadId: uploadId, + }), + ) + .then(() => { + assert.fail('Expected an error listing parts of aborted MPU'); + }) + .catch(err => { + assert.strictEqual(err.name, 'NoSuchUpload'); + done(); + }); }); }); }); @@ -539,8 +542,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should return ServiceUnavailable if MPU deleted directly from AWS ' + - 'and try to complete from S3', done => { + it('should return ServiceUnavailable if MPU deleted directly from AWS ' + 'and try to complete from S3', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { abortMPU(uploadId, getAwsParams(objectKey), () => { @@ -557,8 +559,7 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { + completeMultipartUpload(authInfo, compParams, log, (err, result) => { assert.equal(err, null, `Error completing mpu on AWS: ${err}`); assertMpuCompleteResults(result, objectKey); assertObjOnBackend(awsLocation, objectKey, done); @@ -566,31 +567,25 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should complete a multipart upload on real AWS location with ' + - 'bucketMatch set to false', done => { + it('should complete a multipart upload on real AWS location with ' + 'bucketMatch set to false', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocationMismatch, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { + completeMultipartUpload(authInfo, compParams, log, (err, result) => { assert.equal(err, null, `Error completing mpu on AWS: ${err}`); assertMpuCompleteResults(result, objectKey); - assertObjOnBackend(awsLocationMismatch, - `${bucketName}/${objectKey}`, done); + assertObjOnBackend(awsLocationMismatch, `${bucketName}/${objectKey}`, done); }); }); }); - it('should complete MPU on AWS with same key as object put to file', - done => { + it('should complete MPU on AWS with same key as object put to file', done => { const objectKey = `key-${Date.now()}`; return putObject(fileLocation, objectKey, () => { mpuSetup(awsLocation, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { - assert.equal(err, null, 'Error completing mpu on AWS ' + - `${err}`); + completeMultipartUpload(authInfo, compParams, log, (err, result) => { + assert.equal(err, null, 'Error completing mpu on AWS ' + `${err}`); assertMpuCompleteResults(result, objectKey); assertObjOnBackend(awsLocation, objectKey, done); }); @@ -598,16 +593,13 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should complete MPU on file with same key as object put to AWS', - done => { + it('should complete MPU on file with same key as object put to AWS', done => { const objectKey = `key-${Date.now()}`; putObject(awsLocation, objectKey, () => { mpuSetup(fileLocation, objectKey, uploadId => { const compParams = getCompleteParams(objectKey, uploadId); - completeMultipartUpload(authInfo, compParams, log, - (err, result) => { - assert.equal(err, null, 'Error completing mpu on file ' + - `${err}`); + completeMultipartUpload(authInfo, compParams, log, (err, result) => { + assert.equal(err, null, 'Error completing mpu on file ' + `${err}`); assertMpuCompleteResults(result, objectKey); assertObjOnBackend(fileLocation, objectKey, done); }); @@ -615,28 +607,26 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { }); }); - it('should be successful initiating MPU on AWS with Scality ' + - 'S3 versioning enabled', done => { + it('should be successful initiating MPU on AWS with Scality ' + 'S3 versioning enabled', done => { const objectKey = `key-${Date.now()}`; // putting null version: put obj before versioning configured putObject(awsLocation, objectKey, () => { - const enableVersioningRequest = versioningTestUtils. - createBucketPutVersioningReq(bucketName, 'Enabled'); + const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); bucketPutVersioning(authInfo, enableVersioningRequest, log, err => { - assert.equal(err, null, 'Error enabling bucket versioning: ' + - `${err}`); + assert.equal(err, null, 'Error enabling bucket versioning: ' + `${err}`); const initiateRequest = { bucketName, namespace, objectKey, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': awsLocation }, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': awsLocation, + }, url: `/${objectKey}?uploads`, parsedHost: 'localhost', actionImplicitDenies: false, }; - initiateMultipartUpload(authInfo, initiateRequest, log, - err => { + initiateMultipartUpload(authInfo, initiateRequest, log, err => { assert.strictEqual(err, null); done(); }); @@ -647,7 +637,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { it('should return invalidPart error', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { - const errorBody = '' + + const errorBody = + '' + '' + '1' + `"${awsETag}"` + @@ -668,7 +659,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { it('should return invalidPartOrder error', done => { const objectKey = `key-${Date.now()}`; mpuSetup(awsLocation, objectKey, uploadId => { - const errorBody = '' + + const errorBody = + '' + '' + '2' + `"${awsETag}"` + @@ -693,7 +685,8 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const partRequest3 = new DummyRequest(putPartParam, smallBody); objectPutPart(authInfo, partRequest3, undefined, log, err => { assert.equal(err, null, `Error putting part: ${err}`); - const errorBody = '' + + const errorBody = + '' + '' + '1' + `"${awsETagBigObj}"` + @@ -721,57 +714,54 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; const fileKey = `fileKey-${Date.now()}`; const memKey = `memKey-${Date.now()}`; - async.series([ - cb => mpuSetup(fileLocation, fileKey, - fileUploadId => cb(null, fileUploadId)), - cb => mpuSetup(memLocation, memKey, memUploadId => - cb(null, memUploadId)), - cb => mpuSetup(awsLocation, objectKey, awsUploadId => - cb(null, awsUploadId)), - ], (err, uploadIds) => { - assert.equal(err, null, `Error setting up MPUs: ${err}`); - const listMpuParams = { - bucketName, - namespace, - headers: { host: '/' }, - url: `/${bucketName}?uploads`, - query: {}, - actionImplicitDenies: false, - }; - listMultipartUploads(authInfo, listMpuParams, log, - (err, mpuListXml) => { - assert.equal(err, null, `Error listing MPUs: ${err}`); - parseString(mpuListXml, (err, json) => { - const mpuListing = json.ListMultipartUploadsResult.Upload; - assert.strictEqual(fileKey, mpuListing[0].Key[0]); - assert.strictEqual(uploadIds[0], mpuListing[0].UploadId[0]); - assert.strictEqual(memKey, mpuListing[1].Key[0]); - assert.strictEqual(uploadIds[1], mpuListing[1].UploadId[0]); - assert.strictEqual(objectKey, mpuListing[2].Key[0]); - assert.strictEqual(uploadIds[2], mpuListing[2].UploadId[0]); - const backendsInfo = [ - { backend: fileLocation, key: fileKey, - uploadId: uploadIds[0] }, - { backend: memLocation, key: memKey, - uploadId: uploadIds[1] }, - { backend: 'aws', key: objectKey, - uploadId: uploadIds[2] }, - ]; - abortMultipleMpus(backendsInfo, done); + async.series( + [ + cb => mpuSetup(fileLocation, fileKey, fileUploadId => cb(null, fileUploadId)), + cb => mpuSetup(memLocation, memKey, memUploadId => cb(null, memUploadId)), + cb => mpuSetup(awsLocation, objectKey, awsUploadId => cb(null, awsUploadId)), + ], + (err, uploadIds) => { + assert.equal(err, null, `Error setting up MPUs: ${err}`); + const listMpuParams = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads`, + query: {}, + actionImplicitDenies: false, + }; + listMultipartUploads(authInfo, listMpuParams, log, (err, mpuListXml) => { + assert.equal(err, null, `Error listing MPUs: ${err}`); + parseString(mpuListXml, (err, json) => { + const mpuListing = json.ListMultipartUploadsResult.Upload; + assert.strictEqual(fileKey, mpuListing[0].Key[0]); + assert.strictEqual(uploadIds[0], mpuListing[0].UploadId[0]); + assert.strictEqual(memKey, mpuListing[1].Key[0]); + assert.strictEqual(uploadIds[1], mpuListing[1].UploadId[0]); + assert.strictEqual(objectKey, mpuListing[2].Key[0]); + assert.strictEqual(uploadIds[2], mpuListing[2].UploadId[0]); + const backendsInfo = [ + { backend: fileLocation, key: fileKey, uploadId: uploadIds[0] }, + { backend: memLocation, key: memKey, uploadId: uploadIds[1] }, + { backend: 'aws', key: objectKey, uploadId: uploadIds[2] }, + ]; + abortMultipleMpus(backendsInfo, done); + }); }); - }); - }); + }, + ); }); describe('with mpu initiated on legacy version', () => { beforeEach(function beFn() { this.currentTest.lcObj = config.locationConstraints; - const legacyObj = Object.assign(config.locationConstraints, - { legacy: { + const legacyObj = Object.assign(config.locationConstraints, { + legacy: { type: 'mem', legacyAwsBehavior: true, details: {}, - } }); + }, + }); config.setLocationConstraints(legacyObj); }); @@ -783,40 +773,37 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const objMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - objMD.controllingLocationConstraint = undefined; - objMD.dataStoreName = undefined; - objMD[constants.objectLocationConstraintHeader] = - undefined; - next(null, uploadId, bucketMD, objMD); - }, - (uploadId, bucketMD, objMD, next) => { - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, objMD); - next(null, uploadId); - }, - (uploadId, next) => { - const compParams = getCompleteParams( - objectKey, uploadId); - completeMultipartUpload( - authInfo, compParams, log, next); - }, - (completeRes, resHeaders, next) => { - assertMpuCompleteResults(completeRes, objectKey); - next(); + async.waterfall( + [ + next => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const objMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + objMD.controllingLocationConstraint = undefined; + objMD.dataStoreName = undefined; + objMD[constants.objectLocationConstraintHeader] = undefined; + next(null, uploadId, bucketMD, objMD); + }, + (uploadId, bucketMD, objMD, next) => { + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, objMD); + next(null, uploadId); + }, + (uploadId, next) => { + const compParams = getCompleteParams(objectKey, uploadId); + completeMultipartUpload(authInfo, compParams, log, next); + }, + (completeRes, resHeaders, next) => { + assertMpuCompleteResults(completeRes, objectKey); + next(); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -824,39 +811,37 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const objMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - objMD.controllingLocationConstraint = undefined; - objMD.dataStoreName = undefined; - objMD[constants.objectLocationConstraintHeader] = - undefined; - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, objMD); - next(null, uploadId); - }, - (uploadId, next) => { - const delParams = getDeleteParams(objectKey, uploadId); - multipartDelete(authInfo, delParams, log, - err => next(err, uploadId)); - }, - (uploadId, next) => { - const listParams = getListParams(objectKey, uploadId); - listParts(authInfo, listParams, log, err => { - assert(err.is.NoSuchUpload); - next(); - }); + async.waterfall( + [ + next => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const objMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + objMD.controllingLocationConstraint = undefined; + objMD.dataStoreName = undefined; + objMD[constants.objectLocationConstraintHeader] = undefined; + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, objMD); + next(null, uploadId); + }, + (uploadId, next) => { + const delParams = getDeleteParams(objectKey, uploadId); + multipartDelete(authInfo, delParams, log, err => next(err, uploadId)); + }, + (uploadId, next) => { + const listParams = getListParams(objectKey, uploadId); + listParts(authInfo, listParams, log, err => { + assert(err.is.NoSuchUpload); + next(); + }); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -864,40 +849,38 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const objMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - objMD.controllingLocationConstraint = undefined; - objMD.dataStoreName = undefined; - objMD[constants.objectLocationConstraintHeader] = - undefined; - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, objMD); - next(null, uploadId); - }, - (uploadId, next) => { - const listParams = getListParams(objectKey, uploadId); - listParts(authInfo, listParams, log, (err, res) => { - assert.ifError(err); - assertListResults(res, null, uploadId, objectKey); + async.waterfall( + [ + next => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const objMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + objMD.controllingLocationConstraint = undefined; + objMD.dataStoreName = undefined; + objMD[constants.objectLocationConstraintHeader] = undefined; + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, objMD); next(null, uploadId); - }); - }, - (uploadId, next) => { - const delParams = getDeleteParams(objectKey, uploadId); - multipartDelete(authInfo, delParams, log, - err => next(err, uploadId)); + }, + (uploadId, next) => { + const listParams = getListParams(objectKey, uploadId); + listParts(authInfo, listParams, log, (err, res) => { + assert.ifError(err); + assertListResults(res, null, uploadId, objectKey); + next(null, uploadId); + }); + }, + (uploadId, next) => { + const delParams = getDeleteParams(objectKey, uploadId); + multipartDelete(authInfo, delParams, log, err => next(err, uploadId)); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); @@ -905,70 +888,58 @@ describe('Multipart Upload API with AWS Backend', function mpuTestSuite() { const objectKey = `testkey-${Date.now()}`; mpuSetup('scality-internal-mem', objectKey, uploadId => { const mpuOverviewKey = _getOverviewKey(objectKey, uploadId); - async.waterfall([ - next => { - const copyObjectKey = `copykey-${Date.now()}`; - putObject('scality-internal-mem', copyObjectKey, - () => next(null, copyObjectKey)); - }, - (copyObjectKey, next) => { - const bucketMD = BucketInfo.fromObj( - metadata.buckets.get(bucketName)); - const mpuMD = - metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); - const copyObjMD = - metadata.keyMaps.get(bucketName).get(copyObjectKey); - // remove location constraints to mimic legacy behavior - bucketMD.setLocationConstraint(undefined); - mpuMD.controllingLocationConstraint = undefined; - mpuMD.dataStoreName = undefined; - mpuMD[constants.objectLocationConstraintHeader] = - undefined; - copyObjMD.controllingLocationConstraint = undefined; - copyObjMD.dataStoreName = undefined; - copyObjMD[constants.objectLocationConstraintHeader] = - undefined; - metadata.buckets.set(bucketName, bucketMD); - metadata.keyMaps.get(mpuBucket). - set(mpuOverviewKey, mpuMD); - metadata.keyMaps.get(bucketName). - set(copyObjectKey, copyObjMD); - next(null, uploadId, copyObjectKey); - }, - (uploadId, copyObjectKey, next) => { - const copyParams = - getPartParams(objectKey, uploadId, 3); - objectPutCopyPart(authInfo, copyParams, bucketName, - copyObjectKey, undefined, log, err => { - next(err, uploadId); - }); - }, - (uploadId, next) => { - const listParams = getListParams(objectKey, uploadId); - listParts(authInfo, listParams, log, (err, listRes) => { - assert.ifError(err); - parseString(listRes, (err, json) => { - assert.equal(err, null, - `Error parsing list part results: ${err}`); - assert.strictEqual(json.ListPartsResult. - Part[2].PartNumber[0], '3'); - assert.strictEqual(json.ListPartsResult. - Part[2].ETag[0], `"${awsETag}"`); - assert.strictEqual(json.ListPartsResult. - Part[2].Size[0], '11'); - next(null, uploadId); + async.waterfall( + [ + next => { + const copyObjectKey = `copykey-${Date.now()}`; + putObject('scality-internal-mem', copyObjectKey, () => next(null, copyObjectKey)); + }, + (copyObjectKey, next) => { + const bucketMD = BucketInfo.fromObj(metadata.buckets.get(bucketName)); + const mpuMD = metadata.keyMaps.get(mpuBucket).get(mpuOverviewKey); + const copyObjMD = metadata.keyMaps.get(bucketName).get(copyObjectKey); + // remove location constraints to mimic legacy behavior + bucketMD.setLocationConstraint(undefined); + mpuMD.controllingLocationConstraint = undefined; + mpuMD.dataStoreName = undefined; + mpuMD[constants.objectLocationConstraintHeader] = undefined; + copyObjMD.controllingLocationConstraint = undefined; + copyObjMD.dataStoreName = undefined; + copyObjMD[constants.objectLocationConstraintHeader] = undefined; + metadata.buckets.set(bucketName, bucketMD); + metadata.keyMaps.get(mpuBucket).set(mpuOverviewKey, mpuMD); + metadata.keyMaps.get(bucketName).set(copyObjectKey, copyObjMD); + next(null, uploadId, copyObjectKey); + }, + (uploadId, copyObjectKey, next) => { + const copyParams = getPartParams(objectKey, uploadId, 3); + objectPutCopyPart(authInfo, copyParams, bucketName, copyObjectKey, undefined, log, err => { + next(err, uploadId); }); - }); - }, - (uploadId, next) => { - const delParams = getDeleteParams(objectKey, uploadId); - multipartDelete(authInfo, delParams, log, - err => next(err, uploadId)); + }, + (uploadId, next) => { + const listParams = getListParams(objectKey, uploadId); + listParts(authInfo, listParams, log, (err, listRes) => { + assert.ifError(err); + parseString(listRes, (err, json) => { + assert.equal(err, null, `Error parsing list part results: ${err}`); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], `"${awsETag}"`); + assert.strictEqual(json.ListPartsResult.Part[2].Size[0], '11'); + next(null, uploadId); + }); + }); + }, + (uploadId, next) => { + const delParams = getDeleteParams(objectKey, uploadId); + multipartDelete(authInfo, delParams, log, err => next(err, uploadId)); + }, + ], + err => { + assert.ifError(err); + done(); }, - ], err => { - assert.ifError(err); - done(); - }); + ); }); }); }); diff --git a/tests/multipleBackend/objectCopy.js b/tests/multipleBackend/objectCopy.js index 8e59a1617f..66e147c597 100644 --- a/tests/multipleBackend/objectCopy.js +++ b/tests/multipleBackend/objectCopy.js @@ -6,8 +6,7 @@ const objectPut = require('../../lib/api/objectPut'); const objectCopy = require('../../lib/api/objectCopy'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; const DummyRequest = require('../unit/DummyRequest'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; @@ -19,11 +18,13 @@ const memLocation = 'scality-internal-mem'; const fileLocation = 'scality-internal-file'; function _createBucketPutRequest(bucketName, bucketLoc) { - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; return new DummyRequest({ bucketName, namespace, @@ -56,20 +57,18 @@ function _createObjectPutRequest(bucketName, objectKey, body) { } function copySetup(params, cb) { - const { sourceBucket, sourceLocation, sourceKey, destBucket, - destLocation, body } = params; - const putDestBucketRequest = - _createBucketPutRequest(destBucket, destLocation); - const putSourceBucketRequest = - _createBucketPutRequest(sourceBucket, sourceLocation); - const putSourceObjRequest = _createObjectPutRequest(sourceBucket, - sourceKey, body); - async.series([ - callback => bucketPut(authInfo, putDestBucketRequest, log, callback), - callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), - callback => objectPut(authInfo, putSourceObjRequest, undefined, log, - callback), - ], err => cb(err)); + const { sourceBucket, sourceLocation, sourceKey, destBucket, destLocation, body } = params; + const putDestBucketRequest = _createBucketPutRequest(destBucket, destLocation); + const putSourceBucketRequest = _createBucketPutRequest(sourceBucket, sourceLocation); + const putSourceObjRequest = _createObjectPutRequest(sourceBucket, sourceKey, body); + async.series( + [ + callback => bucketPut(authInfo, putDestBucketRequest, log, callback), + callback => bucketPut(authInfo, putSourceBucketRequest, log, callback), + callback => objectPut(authInfo, putSourceObjRequest, undefined, log, callback), + ], + err => cb(err), + ); } describe('ObjectCopy API with multiple backends', () => { @@ -79,29 +78,30 @@ describe('ObjectCopy API with multiple backends', () => { after(() => cleanup()); - it('object metadata for newly stored object should have dataStoreName ' + - 'if copying to mem based on bucket location', done => { - const params = { - sourceBucket: sourceBucketName, - sourceKey: `sourcekey-${Date.now()}`, - sourceLocation: fileLocation, - body: 'testbody', - destBucket: destBucketName, - destLocation: memLocation, - }; - const destKey = `destkey-${Date.now()}`; - const testObjectCopyRequest = - _createObjectCopyRequest(destBucketName, destKey); - copySetup(params, err => { - assert.strictEqual(err, null, `Error setting up copy: ${err}`); - objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, - params.sourceKey, undefined, log, err => { + it( + 'object metadata for newly stored object should have dataStoreName ' + + 'if copying to mem based on bucket location', + done => { + const params = { + sourceBucket: sourceBucketName, + sourceKey: `sourcekey-${Date.now()}`, + sourceLocation: fileLocation, + body: 'testbody', + destBucket: destBucketName, + destLocation: memLocation, + }; + const destKey = `destkey-${Date.now()}`; + const testObjectCopyRequest = _createObjectCopyRequest(destBucketName, destKey); + copySetup(params, err => { + assert.strictEqual(err, null, `Error setting up copy: ${err}`); + objectCopy(authInfo, testObjectCopyRequest, sourceBucketName, params.sourceKey, undefined, log, err => { assert.strictEqual(err, null, `Error copying: ${err}`); const bucket = metadata.keyMaps.get(params.destBucket); const objMd = bucket.get(destKey); assert.strictEqual(objMd.dataStoreName, memLocation); done(); }); - }); - }); + }); + }, + ); }); diff --git a/tests/multipleBackend/objectPut.js b/tests/multipleBackend/objectPut.js index e76dc24cdc..df31ee0a83 100644 --- a/tests/multipleBackend/objectPut.js +++ b/tests/multipleBackend/objectPut.js @@ -2,8 +2,7 @@ const assert = require('assert'); const async = require('async'); const { storage } = require('arsenal'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const { bucketPut } = require('../../lib/api/bucketPut'); const objectPut = require('../../lib/api/objectPut'); const DummyRequest = require('../unit/DummyRequest'); @@ -24,11 +23,13 @@ const sproxydLocation = 'scality-internal-sproxyd'; const describeSkipIfE2E = process.env.S3_END_TO_END ? describe.skip : describe; function put(bucketLoc, objLoc, requestHost, objectName, cb, errorDescription) { - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; const bucketPutReq = new DummyRequest({ bucketName, namespace, @@ -57,8 +58,7 @@ function put(bucketLoc, objLoc, requestHost, objectName, cb, errorDescription) { testPutObjReq.parsedHost = requestHost; } bucketPut(authInfo, bucketPutReq, log, () => { - objectPut(authInfo, testPutObjReq, undefined, log, (err, - resHeaders) => { + objectPut(authInfo, testPutObjReq, undefined, log, (err, resHeaders) => { if (errorDescription) { assert.strictEqual(err.code, 400); assert(err.is.InvalidArgument); @@ -130,8 +130,7 @@ describeSkipIfE2E('objectPutAPI with multiple backends', function testSuite() { }); function isDataStoredInMem(testCase) { - return testCase.objLoc === memLocation - || (testCase.objLoc === null && testCase.bucketLoc === memLocation); + return testCase.objLoc === memLocation || (testCase.objLoc === null && testCase.bucketLoc === memLocation); } function checkPut(testCase) { @@ -148,28 +147,31 @@ describeSkipIfE2E('objectPutAPI with multiple backends', function testSuite() { putCases.forEach(testCase => { it(`should put an object to ${testCase.name}`, done => { - async.series([ - next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj1', next), - next => { - checkPut(testCase); - // Increase the probability of the first request having released - // the socket, so that it can be reused for the next request. - // This tests how HTTP connection reuse behaves. - setTimeout(next, 10); - }, - // Second put should work as well - next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), - next => { - checkPut(testCase); - setTimeout(next, 10); - }, - // Overwriting PUT - next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), - next => { - checkPut(testCase); - next(); - }, - ], done); + async.series( + [ + next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj1', next), + next => { + checkPut(testCase); + // Increase the probability of the first request having released + // the socket, so that it can be reused for the next request. + // This tests how HTTP connection reuse behaves. + setTimeout(next, 10); + }, + // Second put should work as well + next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), + next => { + checkPut(testCase); + setTimeout(next, 10); + }, + // Overwriting PUT + next => put(testCase.bucketLoc, testCase.objLoc, 'localhost', 'obj2', next), + next => { + checkPut(testCase); + next(); + }, + ], + done, + ); }); }); }); diff --git a/tests/multipleBackend/objectPutCopyPart.js b/tests/multipleBackend/objectPutCopyPart.js index c319ca9df4..0cf3290295 100644 --- a/tests/multipleBackend/objectPutCopyPart.js +++ b/tests/multipleBackend/objectPutCopyPart.js @@ -1,18 +1,12 @@ const assert = require('assert'); const async = require('async'); const { parseString } = require('xml2js'); -const { - S3Client, - ListPartsCommand, - AbortMultipartUploadCommand, -} = require('@aws-sdk/client-s3'); +const { S3Client, ListPartsCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { storage, errors } = require('arsenal'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const { bucketPut } = require('../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../lib/api/initiateMultipartUpload'); const objectPut = require('../../lib/api/objectPut'); const objectPutCopyPart = require('../../lib/api/objectPutCopyPart'); const DummyRequest = require('../unit/DummyRequest'); @@ -62,15 +56,16 @@ function getAwsParamsBucketMismatch(destObjName, uploadId) { return params; } -function copyPutPart(bucketLoc, mpuLoc, srcObjLoc, requestHost, cb, -errorPutCopyPart) { +function copyPutPart(bucketLoc, mpuLoc, srcObjLoc, requestHost, cb, errorPutCopyPart) { const keys = getSourceAndDestKeys(); const { sourceObjName, destObjName } = keys; - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; const bucketPutReq = new DummyRequest({ bucketName, namespace, @@ -90,8 +85,10 @@ errorPutCopyPart) { actionImplicitDenies: false, }; if (mpuLoc) { - initiateReq.headers = { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${mpuLoc}` }; + initiateReq.headers = { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${mpuLoc}`, + }; } if (requestHost) { initiateReq.parsedHost = requestHost; @@ -105,71 +102,78 @@ errorPutCopyPart) { actionImplicitDenies: false, }; if (srcObjLoc) { - sourceObjPutParams.headers = { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${srcObjLoc}` }; + sourceObjPutParams.headers = { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${srcObjLoc}`, + }; } const sourceObjPutReq = new DummyRequest(sourceObjPutParams, body); if (requestHost) { sourceObjPutReq.parsedHost = requestHost; } - async.waterfall([ - next => { - bucketPut(authInfo, bucketPutReq, log, err => { - assert.ifError(err, 'Error putting bucket'); - next(err); - }); - }, - next => { - objectPut(authInfo, sourceObjPutReq, undefined, log, err => - next(err)); - }, - next => { - initiateMultipartUpload(authInfo, initiateReq, log, next); - }, - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${destObjName}`)); - parseString(result, next); - }, - ], - (err, json) => { - // Need to build request in here since do not have - // uploadId until here - assert.ifError(err, 'Error putting source object or initiate MPU'); - const testUploadId = json.InitiateMultipartUploadResult. - UploadId[0]; - const copyPartParams = { - bucketName, - namespace, - objectKey: destObjName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${destObjName}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }; - const copyPartReq = new DummyRequest(copyPartParams); - return objectPutCopyPart(authInfo, copyPartReq, - bucketName, sourceObjName, undefined, log, (err, copyResult) => { - if (errorPutCopyPart) { - assert.strictEqual(err.code, errorPutCopyPart.statusCode); - assert(err.is[errorPutCopyPart.code]); - return cb(); - } - assert.strictEqual(err, null); - return parseString(copyResult, (err, json) => { - assert.equal(err, null, `Error parsing copy result ${err}`); - assert.strictEqual(json.CopyPartResult.ETag[0], - `"${partETag}"`); - assert(json.CopyPartResult.LastModified); - return cb(keys, testUploadId); + async.waterfall( + [ + next => { + bucketPut(authInfo, bucketPutReq, log, err => { + assert.ifError(err, 'Error putting bucket'); + next(err); }); - }); - }); + }, + next => { + objectPut(authInfo, sourceObjPutReq, undefined, log, err => next(err)); + }, + next => { + initiateMultipartUpload(authInfo, initiateReq, log, next); + }, + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${destObjName}`)); + parseString(result, next); + }, + ], + (err, json) => { + // Need to build request in here since do not have + // uploadId until here + assert.ifError(err, 'Error putting source object or initiate MPU'); + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const copyPartParams = { + bucketName, + namespace, + objectKey: destObjName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${destObjName}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }; + const copyPartReq = new DummyRequest(copyPartParams); + return objectPutCopyPart( + authInfo, + copyPartReq, + bucketName, + sourceObjName, + undefined, + log, + (err, copyResult) => { + if (errorPutCopyPart) { + assert.strictEqual(err.code, errorPutCopyPart.statusCode); + assert(err.is[errorPutCopyPart.code]); + return cb(); + } + assert.strictEqual(err, null); + return parseString(copyResult, (err, json) => { + assert.equal(err, null, `Error parsing copy result ${err}`); + assert.strictEqual(json.CopyPartResult.ETag[0], `"${partETag}"`); + assert(json.CopyPartResult.LastModified); + return cb(keys, testUploadId); + }); + }, + ); + }, + ); } function assertPartList(partList, uploadId) { @@ -180,8 +184,7 @@ function assertPartList(partList, uploadId) { assert.strictEqual(partList.Parts[0].Size, 11); } -describeSkipIfE2E('ObjectCopyPutPart API with multiple backends', -function testSuite() { +describeSkipIfE2E('ObjectCopyPutPart API with multiple backends', function testSuite() { this.timeout(60000); beforeEach(() => { @@ -207,8 +210,7 @@ function testSuite() { }); it('should copy part to AWS based on mpu location', done => { - copyPutPart(memLocation, awsLocation, null, 'localhost', - (keys, uploadId) => { + copyPutPart(memLocation, awsLocation, null, 'localhost', (keys, uploadId) => { assert.strictEqual(ds.length, 2); const awsReq = getAwsParams(keys.destObjName, uploadId); s3.send(new ListPartsCommand(awsReq)) @@ -220,8 +222,10 @@ function testSuite() { done(); }) .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); }); }); }); @@ -266,63 +270,71 @@ function testSuite() { done(); }) .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); }); }); }); - it('should copy part an object on AWS location that has ' + - 'bucketMatch equals false to a mpu with a different AWS location', done => { - copyPutPart(null, awsLocation, awsLocationMismatch, 'localhost', - (keys, uploadId) => { - assert.deepStrictEqual(ds, []); - const awsReq = getAwsParams(keys.destObjName, uploadId); - s3.send(new ListPartsCommand(awsReq)) - .then(partList => { - assertPartList(partList, uploadId); - return s3.send(new AbortMultipartUploadCommand(awsReq)); - }) - .then(() => { - done(); - }) - .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); - }); - }); - }); + it( + 'should copy part an object on AWS location that has ' + + 'bucketMatch equals false to a mpu with a different AWS location', + done => { + copyPutPart(null, awsLocation, awsLocationMismatch, 'localhost', (keys, uploadId) => { + assert.deepStrictEqual(ds, []); + const awsReq = getAwsParams(keys.destObjName, uploadId); + s3.send(new ListPartsCommand(awsReq)) + .then(partList => { + assertPartList(partList, uploadId); + return s3.send(new AbortMultipartUploadCommand(awsReq)); + }) + .then(() => { + done(); + }) + .catch(err => { + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); + }); + }); + }, + ); - it('should copy part an object on AWS to a mpu with a different ' + - 'AWS location that has bucketMatch equals false', done => { - copyPutPart(null, awsLocationMismatch, awsLocation, 'localhost', - (keys, uploadId) => { - assert.deepStrictEqual(ds, []); - const awsReq = getAwsParamsBucketMismatch(keys.destObjName, - uploadId); - s3.send(new ListPartsCommand(awsReq)) - .then(partList => { - assertPartList(partList, uploadId); - return s3.send(new AbortMultipartUploadCommand(awsReq)); - }) - .then(() => { - done(); - }) - .catch(err => { - assert.fail(`Error with AWS operations: ${err}. ` + - `You may need to abort MPU with upload ID ${uploadId} manually.`); - }); - }); - }); + it( + 'should copy part an object on AWS to a mpu with a different ' + + 'AWS location that has bucketMatch equals false', + done => { + copyPutPart(null, awsLocationMismatch, awsLocation, 'localhost', (keys, uploadId) => { + assert.deepStrictEqual(ds, []); + const awsReq = getAwsParamsBucketMismatch(keys.destObjName, uploadId); + s3.send(new ListPartsCommand(awsReq)) + .then(partList => { + assertPartList(partList, uploadId); + return s3.send(new AbortMultipartUploadCommand(awsReq)); + }) + .then(() => { + done(); + }) + .catch(err => { + assert.fail( + `Error with AWS operations: ${err}. ` + + `You may need to abort MPU with upload ID ${uploadId} manually.`, + ); + }); + }); + }, + ); // FIXME: does not pass, see CLDSRV-442 - it.skip('should return error 403 AccessDenied copying part to a ' + - 'different AWS location without object READ access', - done => { - copyPutPart(null, awsLocation, awsLocation2, 'localhost', done, - errors.AccessDenied); - }); - + it.skip( + 'should return error 403 AccessDenied copying part to a ' + 'different AWS location without object READ access', + done => { + copyPutPart(null, awsLocation, awsLocation2, 'localhost', done, errors.AccessDenied); + }, + ); it('should copy part to file based on request endpoint', done => { copyPutPart(null, null, memLocation, 'localhost', () => { diff --git a/tests/multipleBackend/objectPutPart.js b/tests/multipleBackend/objectPutPart.js index fcbf29ca1d..a87180a4a3 100644 --- a/tests/multipleBackend/objectPutPart.js +++ b/tests/multipleBackend/objectPutPart.js @@ -2,23 +2,18 @@ const assert = require('assert'); const async = require('async'); const crypto = require('crypto'); const { parseString } = require('xml2js'); -const { S3Client, - ListPartsCommand, - AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); +const { S3Client, ListPartsCommand, AbortMultipartUploadCommand } = require('@aws-sdk/client-s3'); const { storage } = require('arsenal'); const { config } = require('../../lib/Config'); -const { cleanup, DummyRequestLogger, makeAuthInfo } - = require('../unit/helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../unit/helpers'); const { bucketPut } = require('../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../lib/api/initiateMultipartUpload'); const objectPutPart = require('../../lib/api/objectPutPart'); const DummyRequest = require('../unit/DummyRequest'); const mdWrapper = require('../../lib/metadata/wrapper'); const constants = require('../../constants'); -const { getRealAwsConfig } = - require('../functional/aws-node-sdk/test/support/awsConfig'); +const { getRealAwsConfig } = require('../functional/aws-node-sdk/test/support/awsConfig'); const { metadata } = storage.metadata.inMemory.metadata; const { ds } = storage.data.inMemory.datastore; @@ -51,14 +46,15 @@ function _getOverviewKey(objectKey, uploadId) { return `overview${splitter}${objectKey}${splitter}${uploadId}`; } -function putPart(bucketLoc, mpuLoc, requestHost, cb, -errorDescription) { +function putPart(bucketLoc, mpuLoc, requestHost, cb, errorDescription) { const objectName = `objectName-${Date.now()}`; - const post = bucketLoc ? '' + - '' + - `${bucketLoc}` + - '' : ''; + const post = bucketLoc + ? '' + + '' + + `${bucketLoc}` + + '' + : ''; const bucketPutReq = { bucketName, namespace, @@ -78,84 +74,88 @@ errorDescription) { actionImplicitDenies: false, }; if (mpuLoc) { - initiateReq.headers = { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': `${mpuLoc}` }; + initiateReq.headers = { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-meta-scal-location-constraint': `${mpuLoc}`, + }; } if (requestHost) { initiateReq.parsedHost = requestHost; } - async.waterfall([ - next => { - bucketPut(authInfo, bucketPutReq, log, err => { - assert.ifError(err, 'Error putting bucket'); - next(err); - }); - }, - next => { - initiateMultipartUpload(authInfo, initiateReq, log, next); - }, - (result, corsHeaders, next) => { - const mpuKeys = metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuKeys.size, 1); - assert(mpuKeys.keys().next().value - .startsWith(`overview${splitter}${objectName}`)); - parseString(result, next); - }, - ], - (err, json) => { - if (errorDescription) { - assert.strictEqual(err.code, 400); - assert(err.is.InvalidArgument); - assert(err.description.indexOf(errorDescription) > -1); - return cb(); - } - - assert.ifError(err, 'Error initiating MPU'); - - // Need to build request in here since do not have uploadId - // until here - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const partReqParams = { - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - }; - const partReq = new DummyRequest(partReqParams, body1); - return objectPutPart(authInfo, partReq, undefined, log, err => { - assert.strictEqual(err, null); - if (bucketLoc !== awsLocation && mpuLoc !== awsLocation && - bucketLoc !== awsLocationMismatch && - mpuLoc !== awsLocationMismatch) { - const keysInMPUkeyMap = []; - metadata.keyMaps.get(mpuBucket).forEach((val, key) => { - keysInMPUkeyMap.push(key); - }); - const sortedKeyMap = keysInMPUkeyMap.sort(a => { - if (a.slice(0, 8) === 'overview') { - return -1; - } - return 0; + async.waterfall( + [ + next => { + bucketPut(authInfo, bucketPutReq, log, err => { + assert.ifError(err, 'Error putting bucket'); + next(err); }); - const partKey = sortedKeyMap[1]; - const partETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; - assert.strictEqual(keysInMPUkeyMap.length, 2); - assert.strictEqual(partETag, calculatedHash1); + }, + next => { + initiateMultipartUpload(authInfo, initiateReq, log, next); + }, + (result, corsHeaders, next) => { + const mpuKeys = metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuKeys.size, 1); + assert(mpuKeys.keys().next().value.startsWith(`overview${splitter}${objectName}`)); + parseString(result, next); + }, + ], + (err, json) => { + if (errorDescription) { + assert.strictEqual(err.code, 400); + assert(err.is.InvalidArgument); + assert(err.description.indexOf(errorDescription) > -1); + return cb(); } - cb(objectName, testUploadId); - }); - }); + + assert.ifError(err, 'Error initiating MPU'); + + // Need to build request in here since do not have uploadId + // until here + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partReqParams = { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + }; + const partReq = new DummyRequest(partReqParams, body1); + return objectPutPart(authInfo, partReq, undefined, log, err => { + assert.strictEqual(err, null); + if ( + bucketLoc !== awsLocation && + mpuLoc !== awsLocation && + bucketLoc !== awsLocationMismatch && + mpuLoc !== awsLocationMismatch + ) { + const keysInMPUkeyMap = []; + metadata.keyMaps.get(mpuBucket).forEach((val, key) => { + keysInMPUkeyMap.push(key); + }); + const sortedKeyMap = keysInMPUkeyMap.sort(a => { + if (a.slice(0, 8) === 'overview') { + return -1; + } + return 0; + }); + const partKey = sortedKeyMap[1]; + const partETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; + assert.strictEqual(keysInMPUkeyMap.length, 2); + assert.strictEqual(partETag, calculatedHash1); + } + cb(objectName, testUploadId); + }); + }, + ); } function listAndAbort(uploadId, calculatedHash2, objectName, location, done) { - const awsBucket = config.locationConstraints[location]. - details.bucketName; + const awsBucket = config.locationConstraints[location].details.bucketName; const params = { Bucket: awsBucket, Key: objectName, @@ -166,18 +166,22 @@ function listAndAbort(uploadId, calculatedHash2, objectName, location, done) { if (calculatedHash2) { assert.strictEqual(`"${calculatedHash2}"`, data.Parts[0].ETag); } - s3.send(new AbortMultipartUploadCommand(params)).then(() => { - done(); - }).catch(err => { - assert.equal(err, null, `Error aborting MPU: ${err}. ` + - `You must abort MPU with upload ID ${uploadId} manually.`); - done(); - }); + s3.send(new AbortMultipartUploadCommand(params)) + .then(() => { + done(); + }) + .catch(err => { + assert.equal( + err, + null, + `Error aborting MPU: ${err}. ` + `You must abort MPU with upload ID ${uploadId} manually.`, + ); + done(); + }); }); } -describeSkipIfE2E('objectPutPart API with multiple backends', -function testSuite() { +describeSkipIfE2E('objectPutPart API with multiple backends', function testSuite() { this.timeout(50000); beforeEach(() => { @@ -202,40 +206,35 @@ function testSuite() { }); it('should put a part to AWS based on mpu location', done => { - putPart(fileLocation, awsLocation, 'localhost', - (objectName, uploadId) => { + putPart(fileLocation, awsLocation, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); listAndAbort(uploadId, null, objectName, awsLocation, done); }); }); - it('should replace part if two parts uploaded with same part number to AWS', - done => { - putPart(fileLocation, awsLocation, 'localhost', - (objectName, uploadId) => { + it('should replace part if two parts uploaded with same part number to AWS', done => { + putPart(fileLocation, awsLocation, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); const partReqParams = { bucketName, namespace, objectKey: objectName, - headers: { 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-meta-scal-location-constraint': awsLocation }, + headers: { host: `${bucketName}.s3.amazonaws.com`, 'x-amz-meta-scal-location-constraint': awsLocation }, url: `/${objectName}?partNumber=1&uploadId=${uploadId}`, query: { - partNumber: '1', uploadId, + partNumber: '1', + uploadId, }, }; const partReq = new DummyRequest(partReqParams, body2); objectPutPart(authInfo, partReq, undefined, log, err => { assert.equal(err, null, `Error putting second part: ${err}`); - listAndAbort(uploadId, calculatedHash2, - objectName, awsLocation, done); + listAndAbort(uploadId, calculatedHash2, objectName, awsLocation, done); }); }); }); - it('should upload part based on mpu location even if part ' + - 'location constraint is specified ', done => { + it('should upload part based on mpu location even if part ' + 'location constraint is specified ', done => { putPart(fileLocation, memLocation, 'localhost', () => { assert.deepStrictEqual(ds[1].value, body1); done(); @@ -257,29 +256,23 @@ function testSuite() { }); it('should put a part to AWS based on bucket location', done => { - putPart(awsLocation, null, 'localhost', - (objectName, uploadId) => { + putPart(awsLocation, null, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); listAndAbort(uploadId, null, objectName, awsLocation, done); }); }); - it('should put a part to AWS based on bucket location with bucketMatch ' + - 'set to true', done => { - putPart(null, awsLocation, 'localhost', - (objectName, uploadId) => { + it('should put a part to AWS based on bucket location with bucketMatch ' + 'set to true', done => { + putPart(null, awsLocation, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); listAndAbort(uploadId, null, objectName, awsLocation, done); }); }); - it('should put a part to AWS based on bucket location with bucketMatch ' + - 'set to false', done => { - putPart(null, awsLocationMismatch, 'localhost', - (objectName, uploadId) => { + it('should put a part to AWS based on bucket location with bucketMatch ' + 'set to false', done => { + putPart(null, awsLocationMismatch, 'localhost', (objectName, uploadId) => { assert.deepStrictEqual(ds, []); - listAndAbort(uploadId, null, `${bucketName}/${objectName}`, - awsLocationMismatch, done); + listAndAbort(uploadId, null, `${bucketName}/${objectName}`, awsLocationMismatch, done); }); }); @@ -290,28 +283,28 @@ function testSuite() { }); }); - it('should store a part even if the MPU was initiated on legacy version', - done => { - putPart('scality-internal-mem', null, 'localhost', - (objectKey, uploadId) => { + it('should store a part even if the MPU was initiated on legacy version', done => { + putPart('scality-internal-mem', null, 'localhost', (objectKey, uploadId) => { const mputOverviewKey = _getOverviewKey(objectKey, uploadId); - mdWrapper.getObjectMD(mpuBucket, mputOverviewKey, {}, log, - (err, res) => { + mdWrapper.getObjectMD(mpuBucket, mputOverviewKey, {}, log, (err, res) => { // remove location constraint to mimic legacy behvior // eslint-disable-next-line no-param-reassign res.controllingLocationConstraint = undefined; const md5Hash = crypto.createHash('md5'); const bufferBody = Buffer.from(body1); const calculatedHash = md5Hash.update(bufferBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, - query: { partNumber: '1', uploadId }, - calculatedHash, - }, body1); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { partNumber: '1', uploadId }, + calculatedHash, + }, + body1, + ); objectPutPart(authInfo, partRequest, undefined, log, err => { assert.strictEqual(err, null); const keysInMPUkeyMap = []; @@ -325,8 +318,7 @@ function testSuite() { return 0; }); const partKey = sortedKeyMap[1]; - const partETag = metadata.keyMaps.get(mpuBucket) - .get(partKey)['content-md5']; + const partETag = metadata.keyMaps.get(mpuBucket).get(partKey)['content-md5']; assert.strictEqual(keysInMPUkeyMap.length, 2); assert.strictEqual(partETag, calculatedHash); done(); diff --git a/tests/sur/quota.js b/tests/sur/quota.js index 95e84372eb..5c22c03572 100644 --- a/tests/sur/quota.js +++ b/tests/sur/quota.js @@ -1159,7 +1159,6 @@ function multiObjectDelete(bucket, keys, size, callback) { ); }); - // eslint-disable-next-line max-len it('should only evaluate quota and not update inflights for PutObject with the x-scal-s3-version-id header', done => { const bucket = 'quota-test-bucket13'; const key = 'quota-test-object'; diff --git a/tests/unit/api/api.js b/tests/unit/api/api.js index cadd1bef61..81c299120f 100644 --- a/tests/unit/api/api.js +++ b/tests/unit/api/api.js @@ -288,7 +288,7 @@ describe('api.callApiMethod', () => { const body = ''; const headers = { 'content-md5': '1B2M2Y8AsgTpgAmY7PhCfg==', // Wrong MD5 - 'content-length': body.length.toString() + 'content-length': body.length.toString(), }; const requestWithBody = new DummyRequest( @@ -317,14 +317,17 @@ describe('api.callApiMethod', () => { const body = ''; const headers = { 'content-md5': 'x', // Invalid MD5 - 'content-length': body.length.toString() + 'content-length': body.length.toString(), }; - const requestWithBody = new DummyRequest({ - headers, - query: {}, - socket: { remoteAddress: '127.0.0.1', destroy: sandbox.stub() } - }, body); + const requestWithBody = new DummyRequest( + { + headers, + query: {}, + socket: { remoteAddress: '127.0.0.1', destroy: sandbox.stub() }, + }, + body, + ); sandbox.stub(api, method).callsFake(() => { done(new Error(`${method} was called despite bad checksum`)); diff --git a/tests/unit/api/apiUtils/authorization/aclChecks.js b/tests/unit/api/apiUtils/authorization/aclChecks.js index 6bf34850b6..0821d5acc4 100644 --- a/tests/unit/api/apiUtils/authorization/aclChecks.js +++ b/tests/unit/api/apiUtils/authorization/aclChecks.js @@ -1,7 +1,9 @@ const assert = require('assert'); -const { isServiceAccount, getServiceAccountProperties } = - require('../../../../../lib/api/apiUtils/authorization/permissionChecks'); +const { + isServiceAccount, + getServiceAccountProperties, +} = require('../../../../../lib/api/apiUtils/authorization/permissionChecks'); describe('aclChecks', () => { it('should return whether a canonical ID is a service account', () => { @@ -12,15 +14,11 @@ describe('aclChecks', () => { }); it('should return properties of a service account by canonical ID', () => { - assert.strictEqual( - getServiceAccountProperties('abcdefghijkl'), undefined); - assert.strictEqual( - getServiceAccountProperties('abcdefghijkl/notaservice'), undefined); - assert.deepStrictEqual( - getServiceAccountProperties('abcdefghijkl/lifecycle'), {}); - assert.deepStrictEqual( - getServiceAccountProperties('abcdefghijkl/md-ingestion'), { - canReplicate: true, - }); + assert.strictEqual(getServiceAccountProperties('abcdefghijkl'), undefined); + assert.strictEqual(getServiceAccountProperties('abcdefghijkl/notaservice'), undefined); + assert.deepStrictEqual(getServiceAccountProperties('abcdefghijkl/lifecycle'), {}); + assert.deepStrictEqual(getServiceAccountProperties('abcdefghijkl/md-ingestion'), { + canReplicate: true, + }); }); }); diff --git a/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js b/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js index f2573a1b1a..bec694968d 100644 --- a/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js +++ b/tests/unit/api/apiUtils/authorization/prepareRequestContexts.js @@ -1,15 +1,15 @@ const assert = require('assert'); const DummyRequest = require('../../../DummyRequest'); -const prepareRequestContexts = - require('../../../../../lib/api/apiUtils/authorization/prepareRequestContexts.js'); - -const makeRequest = (headers, query) => new DummyRequest({ - headers, - url: '/', - parsedHost: 'localhost', - socket: {}, - query, -}); +const prepareRequestContexts = require('../../../../../lib/api/apiUtils/authorization/prepareRequestContexts.js'); + +const makeRequest = (headers, query) => + new DummyRequest({ + headers, + url: '/', + parsedHost: 'localhost', + socket: {}, + query, + }); const sourceBucket = 'bucketsource'; const sourceObject = 'objectsource'; const sourceVersionId = 'vid1'; @@ -18,76 +18,79 @@ describe('prepareRequestContexts', () => { it('should return s3:DeleteObject if multiObjectDelete method', () => { const apiMethod = 'multiObjectDelete'; const request = makeRequest(); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); const expectedAction = 's3:DeleteObject'; assert.strictEqual(results[0].getAction(), expectedAction); }); - it('should return s3:PutObjectVersion request context action for objectPut method with x-scal-s3-version-id' + - ' header', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-scal-s3-version-id': 'vid', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); + it( + 'should return s3:PutObjectVersion request context action for objectPut method with x-scal-s3-version-id' + + ' header', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-scal-s3-version-id': 'vid', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - it('should return s3:PutObjectVersion request context action for objectPut method with empty x-scal-s3-version-id' + - ' header', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-scal-s3-version-id': '', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); + + it( + 'should return s3:PutObjectVersion request context action for objectPut method with empty x-scal-s3-version-id' + + ' header', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-scal-s3-version-id': '', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); it('should return s3:PutObject request context action for objectPut method and no header', () => { const apiMethod = 'objectPut'; const request = makeRequest({}); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); const expectedAction = 's3:PutObject'; assert.strictEqual(results[0].getAction(), expectedAction); }); - it('should return s3:PutObject and s3:PutObjectTagging actions for objectPut method with' + - ' x-amz-tagging header', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-amz-tagging': 'key1=value1&key2=value2', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + it( + 'should return s3:PutObject and s3:PutObjectTagging actions for objectPut method with' + + ' x-amz-tagging header', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-amz-tagging': 'key1=value1&key2=value2', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:PutObject'; - const expectedAction2 = 's3:PutObjectTagging'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:PutObject'; + const expectedAction2 = 's3:PutObjectTagging'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); it('should return s3:PutObject and s3:PutObjectAcl actions for objectPut method with ACL header', () => { const apiMethod = 'objectPut'; const request = makeRequest({ 'x-amz-acl': 'private', }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 2); const expectedAction1 = 's3:PutObject'; @@ -98,10 +101,8 @@ describe('prepareRequestContexts', () => { it('should return s3:GetObject for headObject', () => { const apiMethod = 'objectHead'; - const request = makeRequest({ - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const request = makeRequest({}); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); assert.strictEqual(results[0].getAction(), 's3:GetObject'); @@ -112,43 +113,46 @@ describe('prepareRequestContexts', () => { const request = makeRequest({ 'x-amz-version-id': '0987654323456789', }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 2); assert.strictEqual(results[0].getAction(), 's3:GetObject'); assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion'); }); - it('should return s3:GetObject and scality:GetObjectArchiveInfo for headObject ' + - 'with x-amz-scal-archive-info header', () => { - const apiMethod = 'objectHead'; - const request = makeRequest({ - 'x-amz-scal-archive-info': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - assert.strictEqual(results[0].getAction(), 's3:GetObject'); - assert.strictEqual(results[1].getAction(), 'scality:GetObjectArchiveInfo'); - }); - - it('should return s3:GetObject, s3:GetObjectVersion and scality:GetObjectArchiveInfo ' + - ' for headObject with x-amz-scal-archive-info header', () => { - const apiMethod = 'objectHead'; - const request = makeRequest({ - 'x-amz-version-id': '0987654323456789', - 'x-amz-scal-archive-info': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + it( + 'should return s3:GetObject and scality:GetObjectArchiveInfo for headObject ' + + 'with x-amz-scal-archive-info header', + () => { + const apiMethod = 'objectHead'; + const request = makeRequest({ + 'x-amz-scal-archive-info': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + assert.strictEqual(results[0].getAction(), 's3:GetObject'); + assert.strictEqual(results[1].getAction(), 'scality:GetObjectArchiveInfo'); + }, + ); + + it( + 'should return s3:GetObject, s3:GetObjectVersion and scality:GetObjectArchiveInfo ' + + ' for headObject with x-amz-scal-archive-info header', + () => { + const apiMethod = 'objectHead'; + const request = makeRequest({ + 'x-amz-version-id': '0987654323456789', + 'x-amz-scal-archive-info': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 3); - assert.strictEqual(results[0].getAction(), 's3:GetObject'); - assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion'); - assert.strictEqual(results[2].getAction(), 'scality:GetObjectArchiveInfo'); - }); + assert.strictEqual(results.length, 3); + assert.strictEqual(results[0].getAction(), 's3:GetObject'); + assert.strictEqual(results[1].getAction(), 's3:GetObjectVersion'); + assert.strictEqual(results[2].getAction(), 'scality:GetObjectArchiveInfo'); + }, + ); it('should return s3:PutObjectRetention with header x-amz-object-lock-mode', () => { const apiMethod = 'objectPut'; @@ -165,45 +169,54 @@ describe('prepareRequestContexts', () => { assert.strictEqual(results[1].getAction(), expectedAction2); }); - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + - 'with header x-amz-bypass-governance-retention', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 3); - const expectedAction1 = 's3:PutObject'; - const expectedAction2 = 's3:PutObjectRetention'; - const expectedAction3 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - assert.strictEqual(results[2].getAction(), expectedAction3); - }); - - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + - 'with header x-amz-bypass-governance-retention with version id specified', () => { - const apiMethod = 'objectPut'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 3); - const expectedAction1 = 's3:PutObject'; - const expectedAction2 = 's3:PutObjectRetention'; - const expectedAction3 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - assert.strictEqual(results[2].getAction(), expectedAction3); - }); + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + + 'with header x-amz-bypass-governance-retention', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest({ + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 3); + const expectedAction1 = 's3:PutObject'; + const expectedAction2 = 's3:PutObjectRetention'; + const expectedAction3 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + assert.strictEqual(results[2].getAction(), expectedAction3); + }, + ); + + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPut ' + + 'with header x-amz-bypass-governance-retention with version id specified', + () => { + const apiMethod = 'objectPut'; + const request = makeRequest( + { + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 3); + const expectedAction1 = 's3:PutObject'; + const expectedAction2 = 's3:PutObjectRetention'; + const expectedAction3 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + assert.strictEqual(results[2].getAction(), expectedAction3); + }, + ); it('should return s3:PutObjectRetention with header x-amz-object-lock-mode for objectPutRetention action', () => { const apiMethod = 'objectPutRetention'; @@ -218,47 +231,55 @@ describe('prepareRequestContexts', () => { assert.strictEqual(results[0].getAction(), expectedAction); }); - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + - 'with header x-amz-bypass-governance-retention', () => { - const apiMethod = 'objectPutRetention'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:PutObjectRetention'; - const expectedAction2 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); - - it('should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + - 'with header x-amz-bypass-governance-retention with version id specified', () => { - const apiMethod = 'objectPutRetention'; - const request = makeRequest({ - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', - 'x-amz-bypass-governance-retention': 'true', - }, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:PutObjectRetention'; - const expectedAction2 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + + 'with header x-amz-bypass-governance-retention', + () => { + const apiMethod = 'objectPutRetention'; + const request = makeRequest({ + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:PutObjectRetention'; + const expectedAction2 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); + + it( + 'should return s3:PutObjectRetention and s3:BypassGovernanceRetention for objectPutRetention ' + + 'with header x-amz-bypass-governance-retention with version id specified', + () => { + const apiMethod = 'objectPutRetention'; + const request = makeRequest( + { + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-retain-until-date': '2021-12-31T23:59:59.000Z', + 'x-amz-bypass-governance-retention': 'true', + }, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:PutObjectRetention'; + const expectedAction2 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); it('should return s3:DeleteObject for objectDelete method', () => { const apiMethod = 'objectDelete'; const request = makeRequest(); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); assert.strictEqual(results[0].getAction(), 's3:DeleteObject'); @@ -266,82 +287,94 @@ describe('prepareRequestContexts', () => { it('should return s3:DeleteObjectVersion for objectDelete method with version id specified', () => { const apiMethod = 'objectDelete'; - const request = makeRequest({}, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const request = makeRequest( + {}, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); assert.strictEqual(results[0].getAction(), 's3:DeleteObjectVersion'); }); // Now it shuld include the bypass header if set - it('should return s3:DeleteObjectVersion and s3:BypassGovernanceRetention for objectDelete method ' + - 'with version id specified and x-amz-bypass-governance-retention header', () => { - const apiMethod = 'objectDelete'; - const request = makeRequest({ - 'x-amz-bypass-governance-retention': 'true', - }, { - versionId: 'vid1', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 2); - const expectedAction1 = 's3:DeleteObjectVersion'; - const expectedAction2 = 's3:BypassGovernanceRetention'; - assert.strictEqual(results[0].getAction(), expectedAction1); - assert.strictEqual(results[1].getAction(), expectedAction2); - }); + it( + 'should return s3:DeleteObjectVersion and s3:BypassGovernanceRetention for objectDelete method ' + + 'with version id specified and x-amz-bypass-governance-retention header', + () => { + const apiMethod = 'objectDelete'; + const request = makeRequest( + { + 'x-amz-bypass-governance-retention': 'true', + }, + { + versionId: 'vid1', + }, + ); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 2); + const expectedAction1 = 's3:DeleteObjectVersion'; + const expectedAction2 = 's3:BypassGovernanceRetention'; + assert.strictEqual(results[0].getAction(), expectedAction1); + assert.strictEqual(results[1].getAction(), expectedAction2); + }, + ); // When there is no version ID, AWS does not return any error if the object // is locked, but creates a delete marker - it('should only return s3:DeleteObject for objectDelete method ' + - 'with x-amz-bypass-governance-retention header and no version id', () => { - const apiMethod = 'objectDelete'; - const request = makeRequest({ - 'x-amz-bypass-governance-retention': 'true', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); - - assert.strictEqual(results.length, 1); - const expectedAction = 's3:DeleteObject'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); - - ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => { - it(`should return s3:PutObjectVersion request context action for ${apiMethod} method ` + - 'with x-scal-s3-version-id header', () => { + it( + 'should only return s3:DeleteObject for objectDelete method ' + + 'with x-amz-bypass-governance-retention header and no version id', + () => { + const apiMethod = 'objectDelete'; const request = makeRequest({ - 'x-scal-s3-version-id': '', + 'x-amz-bypass-governance-retention': 'true', }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; + const expectedAction = 's3:DeleteObject'; assert.strictEqual(results[0].getAction(), expectedAction); - }); + }, + ); - it(`should return s3:PutObjectVersion request context action for ${apiMethod} method` + - 'with empty x-scal-s3-version-id header', () => { - const request = makeRequest({ - 'x-scal-s3-version-id': '', - }); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + ['initiateMultipartUpload', 'objectPutPart', 'completeMultipartUpload'].forEach(apiMethod => { + it( + `should return s3:PutObjectVersion request context action for ${apiMethod} method ` + + 'with x-scal-s3-version-id header', + () => { + const request = makeRequest({ + 'x-scal-s3-version-id': '', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); - assert.strictEqual(results.length, 1); - const expectedAction = 's3:PutObjectVersion'; - assert.strictEqual(results[0].getAction(), expectedAction); - }); + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); + + it( + `should return s3:PutObjectVersion request context action for ${apiMethod} method` + + 'with empty x-scal-s3-version-id header', + () => { + const request = makeRequest({ + 'x-scal-s3-version-id': '', + }); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); + + assert.strictEqual(results.length, 1); + const expectedAction = 's3:PutObjectVersion'; + assert.strictEqual(results[0].getAction(), expectedAction); + }, + ); it(`should return s3:PutObject request context action for ${apiMethod} method and no header`, () => { const request = makeRequest({}); - const results = prepareRequestContexts(apiMethod, request, sourceBucket, - sourceObject, sourceVersionId); + const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); assert.strictEqual(results.length, 1); const expectedAction = 's3:PutObject'; @@ -463,7 +496,7 @@ describe('prepareRequestContexts', () => { const apiMethod = 'objectGetAttributes'; const request = makeRequest( { 'x-amz-object-attributes': 'x-amz-meta-department' }, - { versionId: '0987654323456789' } + { versionId: '0987654323456789' }, ); const results = prepareRequestContexts(apiMethod, request, sourceBucket, sourceObject, sourceVersionId); diff --git a/tests/unit/api/apiUtils/expirationHeaders.js b/tests/unit/api/apiUtils/expirationHeaders.js index 41fa1af48c..07824e4854 100644 --- a/tests/unit/api/apiUtils/expirationHeaders.js +++ b/tests/unit/api/apiUtils/expirationHeaders.js @@ -1,25 +1,19 @@ const assert = require('assert'); const { LifecycleDateTime } = require('arsenal').s3middleware.lifecycleHelpers; - -const { - generateExpirationHeaders, -} = require('../../../../lib/api/apiUtils/object/expirationHeaders'); +const { generateExpirationHeaders } = require('../../../../lib/api/apiUtils/object/expirationHeaders'); const datetime = new LifecycleDateTime(); const objectDate = 'Fri, 21 Dec 2012 00:00:00 GMT'; const expectedDaysExpiryDate = 'Sat, 22 Dec 2012 00:00:00 GMT'; const expectedDateExpiryDate = 'Mon, 24 Dec 2012 00:00:00 GMT'; - const lifecycleExpirationDays = { rules: [ { ruleID: 'test-days', ruleStatus: 'Enabled', - actions: [ - { actionName: 'Expiration', days: 1 }, - ], + actions: [{ actionName: 'Expiration', days: 1 }], prefix: '', }, ], @@ -31,13 +25,9 @@ const lifecycleExpirationTags = { ruleID: 'test-tags', ruleStatus: 'Enabled', filters: { - tags: [ - { key: 'key1', val: 'val1' }, - ], + tags: [{ key: 'key1', val: 'val1' }], }, - actions: [ - { actionName: 'Expiration', days: 1 }, - ], + actions: [{ actionName: 'Expiration', days: 1 }], }, ], }; @@ -47,9 +37,7 @@ const lifecycleExpirationDate = { { ruleID: 'test-date', ruleStatus: 'Enabled', - actions: [ - { actionName: 'Expiration', date: 'Mon, 24 Dec 2012 00:00:00 GMT' }, - ], + actions: [{ actionName: 'Expiration', date: 'Mon, 24 Dec 2012 00:00:00 GMT' }], prefix: '', }, ], @@ -60,9 +48,7 @@ const lifecycleExpirationMPU = { { ruleID: 'test-mpu', ruleStatus: 'Enabled', - actions: [ - { actionName: 'AbortIncompleteMultipartUpload', days: 1 }, - ], + actions: [{ actionName: 'AbortIncompleteMultipartUpload', days: 1 }], prefix: '', }, ], @@ -172,7 +158,9 @@ describe('generateExpirationHeaders', () => { ], ]; - tests.forEach(([msg, params, expected]) => it(msg, () => { - assert.deepStrictEqual(generateExpirationHeaders(params, datetime), expected); - })); + tests.forEach(([msg, params, expected]) => + it(msg, () => { + assert.deepStrictEqual(generateExpirationHeaders(params, datetime), expected); + }), + ); }); diff --git a/tests/unit/api/apiUtils/getNotificationConfiguration.js b/tests/unit/api/apiUtils/getNotificationConfiguration.js index d5a531421c..8d226edb2e 100644 --- a/tests/unit/api/apiUtils/getNotificationConfiguration.js +++ b/tests/unit/api/apiUtils/getNotificationConfiguration.js @@ -3,8 +3,7 @@ const sinon = require('sinon'); const { config } = require('../../../../lib/Config'); const errors = require('arsenal').errors; -const getNotificationConfiguration = - require('../../../../lib/api/apiUtils/bucket/getNotificationConfiguration'); +const getNotificationConfiguration = require('../../../../lib/api/apiUtils/bucket/getNotificationConfiguration'); const parsedXml = { NotificationConfiguration: { @@ -15,18 +14,18 @@ const parsedXml = { Queue: ['arn:scality:bucketnotif:::target1'], }, ], - } + }, }; const expectedConfig = { queueConfig: [ { - events: ['s3:ObjectCreated:*'], - queueArn: 'arn:scality:bucketnotif:::target1', - id: 'notification-id', - filterRules: undefined - } - ] + events: ['s3:ObjectCreated:*'], + queueArn: 'arn:scality:bucketnotif:::target1', + id: 'notification-id', + filterRules: undefined, + }, + ], }; const destination1 = [ @@ -34,7 +33,7 @@ const destination1 = [ resource: 'target1', type: 'dummy', host: 'localhost:6000', - } + }, ]; const destinations2 = [ @@ -42,7 +41,7 @@ const destinations2 = [ resource: 'target2', type: 'dummy', host: 'localhost:6000', - } + }, ]; describe('getNotificationConfiguration', () => { @@ -58,7 +57,7 @@ describe('getNotificationConfiguration', () => { it('should return empty notification configuration', done => { sinon.stub(config, 'bucketNotificationDestinations').value(destination1); const notifConfig = getNotificationConfiguration({ - NotificationConfiguration: {} + NotificationConfiguration: {}, }); assert.deepEqual(notifConfig, {}); return done(); @@ -76,10 +75,12 @@ describe('getNotificationConfiguration', () => { const notifConfig = getNotificationConfiguration(parsedXml); assert.deepEqual(notifConfig.error, errors.InvalidArgument); const invalidArguments = notifConfig.error.metadata.get('invalidArguments'); - assert.deepEqual(invalidArguments, [{ - ArgumentName: 'arn:scality:bucketnotif:::target1', - ArgumentValue: 'The destination queue does not exist', - }]); + assert.deepEqual(invalidArguments, [ + { + ArgumentName: 'arn:scality:bucketnotif:::target1', + ArgumentValue: 'The destination queue does not exist', + }, + ]); return done(); }); }); diff --git a/tests/unit/api/apiUtils/locationKeysHaveChanged.js b/tests/unit/api/apiUtils/locationKeysHaveChanged.js index 39d16712e9..7c0575ed56 100644 --- a/tests/unit/api/apiUtils/locationKeysHaveChanged.js +++ b/tests/unit/api/apiUtils/locationKeysHaveChanged.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const locationKeysHaveChanged = - require('../../../../lib/api/apiUtils/object/locationKeysHaveChanged'); +const locationKeysHaveChanged = require('../../../../lib/api/apiUtils/object/locationKeysHaveChanged'); describe('Check if location keys have changed between object locations', () => { it('should return true for no match ', () => { diff --git a/tests/unit/api/apiUtils/object/storeObject.js b/tests/unit/api/apiUtils/object/storeObject.js index 9270719c0b..fc5bfb0e2e 100644 --- a/tests/unit/api/apiUtils/object/storeObject.js +++ b/tests/unit/api/apiUtils/object/storeObject.js @@ -206,7 +206,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should wait for finish before validating when checksumedStream is not yet writableFinished after data.put', done => { let capturedStream; putStub.callsFake((cipher, stream, size, ctx, backend, log2, cb) => { @@ -233,7 +232,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should delete stored data and call cb with the error when checksumedStream emits error after data.put', done => { batchDeleteSucceeds(); let capturedStream; @@ -364,7 +362,6 @@ describe('dataStore', () => { }); describe('x-amz-content-sha256 body validation', () => { - // eslint-disable-next-line max-len it('should call cb with XAmzContentSHA256Mismatch and delete stored data when the hash does not match', done => { batchDeleteSucceeds(); putSucceeds(); @@ -406,7 +403,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should call cb with XAmzContentSHA256Mismatch when the hash mismatches and batchDelete also fails', done => { batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.InternalError)); putSucceeds(); @@ -516,7 +512,6 @@ describe('dataStore', () => { }); }); - // eslint-disable-next-line max-len it('should call cb with stream error when checksumedStream errors after data.put and batchDelete also fails', done => { batchDeleteStub.callsFake((keys, a, b, log2, cb) => cb(errors.BadRequest)); let capturedStream; diff --git a/tests/unit/api/apiUtils/objectLockHelpers.js b/tests/unit/api/apiUtils/objectLockHelpers.js index 285efc9c8f..9769fb8a5b 100644 --- a/tests/unit/api/apiUtils/objectLockHelpers.js +++ b/tests/unit/api/apiUtils/objectLockHelpers.js @@ -16,14 +16,54 @@ const mockOwnerDisplayName = 'accountDisplayName'; const mockCreationDate = new Date().toJSON(); const bucketInfo = new BucketInfo( - mockName, mockOwner, mockOwnerDisplayName, mockCreationDate, - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, true); + mockName, + mockOwner, + mockOwnerDisplayName, + mockCreationDate, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + true, +); const objLockDisabledBucketInfo = new BucketInfo( - mockName, mockOwner, mockOwnerDisplayName, mockCreationDate, - null, null, null, null, null, null, null, null, null, null, - null, null, null, null, null, null, null, false); + mockName, + mockOwner, + mockOwnerDisplayName, + mockCreationDate, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + false, +); const log = new DummyRequestLogger(); @@ -33,13 +73,12 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-retain-until-date': '2050-10-12', 'x-amz-object-lock-mode': 'COMPLIANCE', }; - const objectLockValidationError - = validateHeaders(objLockDisabledBucketInfo, headers, log); + const objectLockValidationError = validateHeaders(objLockDisabledBucketInfo, headers, log); const expectedError = errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing ObjectLockConfiguration'); + 'Bucket is missing ObjectLockConfiguration', + ); assert.strictEqual(objectLockValidationError.is.InvalidRequest, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should pass with valid retention headers', () => { @@ -47,8 +86,7 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-retain-until-date': '2050-10-12', 'x-amz-object-lock-mode': 'COMPLIANCE', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -56,8 +94,7 @@ describe('objectLockHelpers: validateHeaders', () => { const headers = { 'x-amz-object-lock-legal-hold': 'ON', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -65,8 +102,7 @@ describe('objectLockHelpers: validateHeaders', () => { const headers = { 'x-amz-object-lock-legal-hold': 'OFF', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -76,8 +112,7 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-mode': 'GOVERNANCE', 'x-amz-object-lock-legal-hold': 'ON', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError, null); }); @@ -85,28 +120,24 @@ describe('objectLockHelpers: validateHeaders', () => { const headers = { 'x-amz-object-lock-retain-until-date': '2005-10-12', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + - 'must both be supplied'); + 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + 'must both be supplied', + ); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with missing object-lock-retain-until-date header', () => { const headers = { 'x-amz-object-lock-mode': 'GOVERNANCE', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + - 'must both be supplied'); + 'x-amz-object-lock-retain-until-date and x-amz-object-lock-mode ' + 'must both be supplied', + ); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with past retention date header', () => { @@ -115,25 +146,23 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-mode': 'COMPLIANCE', }; const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'The retain until date must be in the future!'); - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + 'The retain until date must be in the future!', + ); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with invalid legal hold header', () => { const headers = { 'x-amz-object-lock-legal-hold': 'on', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'Legal hold status must be one of "ON", "OFF"'); + 'Legal hold status must be one of "ON", "OFF"', + ); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); it('should fail with invalid retention period header', () => { @@ -141,13 +170,10 @@ describe('objectLockHelpers: validateHeaders', () => { 'x-amz-object-lock-retain-until-date': '2050-10-12', 'x-amz-object-lock-mode': 'Governance', }; - const objectLockValidationError - = validateHeaders(bucketInfo, headers, log); - const expectedError = errorInstances.InvalidArgument.customizeDescription( - 'Unknown wormMode directive'); + const objectLockValidationError = validateHeaders(bucketInfo, headers, log); + const expectedError = errorInstances.InvalidArgument.customizeDescription('Unknown wormMode directive'); assert.strictEqual(objectLockValidationError.is.InvalidArgument, true); - assert.strictEqual(objectLockValidationError.description, - expectedError.description); + assert.strictEqual(objectLockValidationError.description, expectedError.description); }); }); @@ -158,11 +184,9 @@ describe('objectLockHelpers: calculateRetainUntilDate', () => { days: 90, }; const date = moment(); - const expectedRetainUntilDate - = date.add(mockConfigWithDays.days * 86400000, 'ms'); + const expectedRetainUntilDate = date.add(mockConfigWithDays.days * 86400000, 'ms'); const retainUntilDate = calculateRetainUntilDate(mockConfigWithDays); - assert.strictEqual(retainUntilDate.slice(0, 16), - expectedRetainUntilDate.toISOString().slice(0, 16)); + assert.strictEqual(retainUntilDate.slice(0, 16), expectedRetainUntilDate.toISOString().slice(0, 16)); }); it('should calculate retainUntilDate for config with years', () => { @@ -171,11 +195,9 @@ describe('objectLockHelpers: calculateRetainUntilDate', () => { years: 3, }; const date = moment(); - const expectedRetainUntilDate - = date.add(mockConfigWithYears.years * 365 * 86400000, 'ms'); + const expectedRetainUntilDate = date.add(mockConfigWithYears.years * 365 * 86400000, 'ms'); const retainUntilDate = calculateRetainUntilDate(mockConfigWithYears); - assert.strictEqual(retainUntilDate.slice(0, 16), - expectedRetainUntilDate.toISOString().slice(0, 16)); + assert.strictEqual(retainUntilDate.slice(0, 16), expectedRetainUntilDate.toISOString().slice(0, 16)); }); }); @@ -269,7 +291,6 @@ describe('objectLockHelpers: compareObjectLockInformation', () => { }); }); - const pastDate = moment().subtract(1, 'days'); const futureDate = moment().add(100, 'days'); @@ -608,46 +629,50 @@ describe('objectLockHelpers: ObjectLockInfo', () => { }); }); - describe('isExpired: ', () => isExpiredTestCases.forEach(testCase => { - const objLockInfo = new ObjectLockInfo({ date: testCase.date }); - it(testCase.desc, () => assert.strictEqual(objLockInfo.isExpired(), testCase.expected)); - })); - - describe('isLocked: ', () => isLockedTestCases.forEach(testCase => { - describe(`${testCase.desc}`, () => { - it(`should show policy as ${testCase.expected ? '' : 'not'} locked without legal hold`, () => { - const objLockInfo = new ObjectLockInfo(testCase.policy); - assert.strictEqual(objLockInfo.isLocked(), testCase.expected); - }); - - // legal hold should show as locked regardless of policy - it('should show policy as locked with legal hold', () => { - const policy = Object.assign({}, testCase.policy, { legalHold: true }); - const objLockInfo = new ObjectLockInfo(policy); - assert.strictEqual(objLockInfo.isLocked(), true); + describe('isExpired: ', () => + isExpiredTestCases.forEach(testCase => { + const objLockInfo = new ObjectLockInfo({ date: testCase.date }); + it(testCase.desc, () => assert.strictEqual(objLockInfo.isExpired(), testCase.expected)); + })); + + describe('isLocked: ', () => + isLockedTestCases.forEach(testCase => { + describe(`${testCase.desc}`, () => { + it(`should show policy as ${testCase.expected ? '' : 'not'} locked without legal hold`, () => { + const objLockInfo = new ObjectLockInfo(testCase.policy); + assert.strictEqual(objLockInfo.isLocked(), testCase.expected); + }); + + // legal hold should show as locked regardless of policy + it('should show policy as locked with legal hold', () => { + const policy = Object.assign({}, testCase.policy, { legalHold: true }); + const objLockInfo = new ObjectLockInfo(policy); + assert.strictEqual(objLockInfo.isLocked(), true); + }); }); - }); - })); + })); - describe('canModifyPolicy: ', () => policyChangeTestCases.forEach(testCase => { - describe(testCase.desc, () => { - const objLockInfo = new ObjectLockInfo(testCase.from); - it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying the policy without bypass`, - () => assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to), testCase.allowed)); + describe('canModifyPolicy: ', () => + policyChangeTestCases.forEach(testCase => { + describe(testCase.desc, () => { + const objLockInfo = new ObjectLockInfo(testCase.from); + it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying the policy without bypass`, () => + assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to), testCase.allowed)); - it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying the policy with bypass`, - () => assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to, true), testCase.allowedWithBypass)); - }); - })); + it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying the policy with bypass`, () => + assert.strictEqual(objLockInfo.canModifyPolicy(testCase.to, true), testCase.allowedWithBypass)); + }); + })); - describe('canModifyObject: ', () => canModifyObjectTestCases.forEach(testCase => { - describe(testCase.desc, () => { - const objLockInfo = new ObjectLockInfo(testCase.policy); - it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying object without bypass`, - () => assert.strictEqual(objLockInfo.canModifyObject(), testCase.allowed)); + describe('canModifyObject: ', () => + canModifyObjectTestCases.forEach(testCase => { + describe(testCase.desc, () => { + const objLockInfo = new ObjectLockInfo(testCase.policy); + it(`should ${testCase.allowed ? 'allow' : 'deny'} modifying object without bypass`, () => + assert.strictEqual(objLockInfo.canModifyObject(), testCase.allowed)); - it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying object with bypass`, - () => assert.strictEqual(objLockInfo.canModifyObject(true), testCase.allowedWithBypass)); - }); - })); + it(`should ${testCase.allowedWithBypass ? 'allow' : 'deny'} modifying object with bypass`, () => + assert.strictEqual(objLockInfo.canModifyObject(true), testCase.allowedWithBypass)); + }); + })); }); diff --git a/tests/unit/api/apiUtils/permissionChecks.js b/tests/unit/api/apiUtils/permissionChecks.js index 03005ed541..15d8427832 100644 --- a/tests/unit/api/apiUtils/permissionChecks.js +++ b/tests/unit/api/apiUtils/permissionChecks.js @@ -1,8 +1,13 @@ const assert = require('assert'); -const { isLifecycleSession, checkBucketPolicyResult, checkBucketPolicy, - isBucketAuthorized, isObjAuthorized, evaluateBucketPolicyWithIAM } = - require('../../../../lib/api/apiUtils/authorization/permissionChecks.js'); +const { + isLifecycleSession, + checkBucketPolicyResult, + checkBucketPolicy, + isBucketAuthorized, + isObjAuthorized, + evaluateBucketPolicyWithIAM, +} = require('../../../../lib/api/apiUtils/authorization/permissionChecks.js'); const { DummyRequestLogger } = require('../../helpers'); const stubLog = new DummyRequestLogger(); @@ -48,7 +53,6 @@ describe('authInfoHelper', () => { }); describe('checkBucketPolicy Principal logic', () => { - /* eslint-disable max-len */ const tests = [ { description: 'bucket owner with same canonicalID as requesters should return ALLOW', @@ -86,12 +90,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -106,7 +106,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester share the same account, principal account ID, Allow policy should return ALLOW', + description: + 'bucket owner and requester share the same account, principal account ID, Allow policy should return ALLOW', policy: { Statement: [ { @@ -115,12 +116,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '123456789012', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -144,12 +141,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -164,7 +157,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, Allow policy should return CROSS ACCOUNT', + description: + "bucket owner and requester don't share the same account, Allow policy should return CROSS ACCOUNT", policy: { Statement: [ { @@ -173,12 +167,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -193,7 +183,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, principal account ID, Allow policy should return CROSS ACCOUNT', + description: + "bucket owner and requester don't share the same account, principal account ID, Allow policy should return CROSS ACCOUNT", policy: { Statement: [ { @@ -202,12 +193,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '123456789012', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -222,7 +209,7 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, Deny policy should return DENY', + description: "bucket owner and requester don't share the same account, Deny policy should return DENY", policy: { Statement: [ { @@ -231,12 +218,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -251,7 +234,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, requester is root, Allow policy should return ALLOW', + description: + "bucket owner and requester don't share the same account, requester is root, Allow policy should return ALLOW", policy: { Statement: [ { @@ -260,12 +244,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -280,7 +260,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, requester is root, Deny policy should return DENY', + description: + "bucket owner and requester don't share the same account, requester is root, Deny policy should return DENY", policy: { Statement: [ { @@ -289,12 +270,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:root', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -309,7 +286,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, requester and principal are users, Allow policy should return CROSS_ACCOUNT', + description: + "bucket owner and requester don't share the same account, requester and principal are users, Allow policy should return CROSS_ACCOUNT", policy: { Statement: [ { @@ -318,12 +296,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:user/testuser', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -338,7 +312,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, requester and principal are users, Deny policy should return EXPLICIT_DENY', + description: + "bucket owner and requester don't share the same account, requester and principal are users, Deny policy should return EXPLICIT_DENY", policy: { Statement: [ { @@ -347,12 +322,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: 'arn:aws:iam::123456789012:user/testuser', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -367,7 +338,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.EXPLICIT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, wildcard "*" principal, Allow policy should return CROSS_ACCOUNT_ALLOW', + description: + 'bucket owner and requester don\'t share the same account, wildcard "*" principal, Allow policy should return CROSS_ACCOUNT_ALLOW', policy: { Statement: [ { @@ -376,12 +348,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '*', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -396,7 +364,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester share the same account, wildcard "*" principal, Allow policy should return ALLOW', + description: + 'bucket owner and requester share the same account, wildcard "*" principal, Allow policy should return ALLOW', policy: { Statement: [ { @@ -405,12 +374,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { AWS: '*', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -425,19 +390,16 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester share the same account, wildcard "*" principal, string typeof principal , Allow policy should return ALLOW', + description: + 'bucket owner and requester share the same account, wildcard "*" principal, string typeof principal , Allow policy should return ALLOW', policy: { Statement: [ { Sid: 'Example permissions', Effect: 'Allow', Principal: '*', - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -452,19 +414,16 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, wildcard "*" principal, string typeof principal, Allow policy should return CROSS_ACCOUNT_ALLOW', + description: + 'bucket owner and requester don\'t share the same account, wildcard "*" principal, string typeof principal, Allow policy should return CROSS_ACCOUNT_ALLOW', policy: { Statement: [ { Sid: 'Example permissions', Effect: 'Allow', Principal: '*', - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -479,7 +438,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.CROSS_ACCOUNT_ALLOW, }, { - description: 'bucket owner and requester don\'t share the same account, no bucket policy for user, canonical user principal, Allow policy should return DEFAULT_DENY', + description: + "bucket owner and requester don't share the same account, no bucket policy for user, canonical user principal, Allow policy should return DEFAULT_DENY", policy: { Statement: [ { @@ -488,12 +448,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -508,7 +464,8 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.DEFAULT_DENY, }, { - description: 'bucket owner and requester don\'t share the same account, canonical user principal, Allow policy should return CROSS_ACCOUNT_ALLOW', + description: + "bucket owner and requester don't share the same account, canonical user principal, Allow policy should return CROSS_ACCOUNT_ALLOW", policy: { Statement: [ { @@ -517,12 +474,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -546,12 +499,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, { Sid: 'Example permissions', @@ -559,12 +508,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -588,12 +533,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, { Sid: 'Example permissions', @@ -601,12 +542,8 @@ describe('checkBucketPolicy Principal logic', () => { Principal: { CanonicalUser: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -628,12 +565,8 @@ describe('checkBucketPolicy Principal logic', () => { Sid: 'Example permissions', Effect: 'Allow', Principal: '*', - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -655,12 +588,8 @@ describe('checkBucketPolicy Principal logic', () => { Sid: 'Example permissions', Effect: 'Allow', Principal: { AWS: '*' }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -682,12 +611,8 @@ describe('checkBucketPolicy Principal logic', () => { Sid: 'Example permissions', Effect: 'Allow', Principal: { CanonicalUser: '*' }, - Action: [ - 's3:*', - ], - Resource: [ - 'arn:aws:s3:::amzn-s3-demo-bucket', - ], + Action: ['s3:*'], + Resource: ['arn:aws:s3:::amzn-s3-demo-bucket'], }, ], }, @@ -702,12 +627,19 @@ describe('checkBucketPolicy Principal logic', () => { expectedResult: checkBucketPolicyResult.ALLOW, }, ]; - /* eslint-enable max-len */ tests.forEach(t => { it(t.description, () => { - const res = checkBucketPolicy(t.policy, t.requestType, t.canonicalID, t.arn, t.bucketOwner, - t.log, t.request, t.actionImplicitDenies); + const res = checkBucketPolicy( + t.policy, + t.requestType, + t.canonicalID, + t.arn, + t.bucketOwner, + t.log, + t.request, + t.actionImplicitDenies, + ); assert.equal(res, t.expectedResult); }); }); @@ -745,8 +677,7 @@ describe('aclRequired field in isBucketAuthorized', () => { const request = makeRequest(); const bucket = makeBucket(ownerCanonicalId); const authInfo = makeAuthInfo(ownerCanonicalId, 'arn:aws:iam::123456789012:/owner/'); - isBucketAuthorized(bucket, 'bucketGet', ownerCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', ownerCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); @@ -754,8 +685,7 @@ describe('aclRequired field in isBucketAuthorized', () => { const request = makeRequest(); const bucket = makeBucket(ownerCanonicalId, { READ: [otherCanonicalId] }); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, 'Yes'); }); @@ -764,34 +694,44 @@ describe('aclRequired field in isBucketAuthorized', () => { // Use an IAM user in the bucket owner's account so principal match is OK (not CROSS_ACCOUNT) // and the owner early-return doesn't fire (requester is IAM user, not account root) const iamUserCanonicalId = ownerCanonicalId; - const bucket = makeBucket(ownerCanonicalId, {}, { - Statement: [{ - Effect: 'Allow', - Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, - Action: ['s3:ListBucket'], - Resource: ['arn:aws:s3:::test-bucket'], - }], - }); + const bucket = makeBucket( + ownerCanonicalId, + {}, + { + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, + Action: ['s3:ListBucket'], + Resource: ['arn:aws:s3:::test-bucket'], + }, + ], + }, + ); const authInfo = makeAuthInfo(iamUserCanonicalId, 'arn:aws:iam::123456789012:user/iamuser', true); - isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); it('should not set aclRequired when bucket policy explicitly denies', () => { const request = makeRequest(); const iamUserCanonicalId = ownerCanonicalId; - const bucket = makeBucket(ownerCanonicalId, { READ: [iamUserCanonicalId] }, { - Statement: [{ - Effect: 'Deny', - Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, - Action: ['s3:ListBucket'], - Resource: ['arn:aws:s3:::test-bucket'], - }], - }); + const bucket = makeBucket( + ownerCanonicalId, + { READ: [iamUserCanonicalId] }, + { + Statement: [ + { + Effect: 'Deny', + Principal: { AWS: 'arn:aws:iam::123456789012:user/iamuser' }, + Action: ['s3:ListBucket'], + Resource: ['arn:aws:s3:::test-bucket'], + }, + ], + }, + ); const authInfo = makeAuthInfo(iamUserCanonicalId, 'arn:aws:iam::123456789012:user/iamuser', true); - isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', iamUserCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); @@ -800,17 +740,22 @@ describe('aclRequired field in isBucketAuthorized', () => { // Policy grants PutObject to a different principal — nothing matches // the bucketGet request from otherCanonicalId, so checkBucketPolicy // returns DEFAULT_DENY and falls back to ACL evaluation. - const bucket = makeBucket(ownerCanonicalId, { READ: [otherCanonicalId] }, { - Statement: [{ - Effect: 'Allow', - Principal: { AWS: 'arn:aws:iam::111111111111:root' }, - Action: ['s3:PutObject'], - Resource: ['arn:aws:s3:::test-bucket/*'], - }], - }); + const bucket = makeBucket( + ownerCanonicalId, + { READ: [otherCanonicalId] }, + { + Statement: [ + { + Effect: 'Allow', + Principal: { AWS: 'arn:aws:iam::111111111111:root' }, + Action: ['s3:PutObject'], + Resource: ['arn:aws:s3:::test-bucket/*'], + }, + ], + }, + ); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, request, { bucketGet: false }); assert.strictEqual(request.serverAccessLog.aclRequired, 'Yes'); }); @@ -819,8 +764,7 @@ describe('aclRequired field in isBucketAuthorized', () => { const bucket = makeBucket(ownerCanonicalId, { READ: [otherCanonicalId] }); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); assert.doesNotThrow(() => { - isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, - request, { bucketGet: false }); + isBucketAuthorized(bucket, 'bucketGet', otherCanonicalId, authInfo, stubLog, request, { bucketGet: false }); }); }); }); @@ -873,8 +817,9 @@ describe('aclRequired field in isObjAuthorized', () => { const bucket = makeBucket(ownerCanonicalId); const objectMD = makeObjectMD(otherCanonicalId); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:/account/'); - isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, - request, { objectGet: false }); + isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, request, { + objectGet: false, + }); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); @@ -884,8 +829,9 @@ describe('aclRequired field in isObjAuthorized', () => { const objectMD = makeObjectMD(ownerCanonicalId); objectMD.acl.READ = [otherCanonicalId]; const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, - request, { objectGet: false }); + isObjAuthorized(bucket, objectMD, 'objectGet', otherCanonicalId, authInfo, stubLog, request, { + objectGet: false, + }); assert.strictEqual(request.serverAccessLog.aclRequired, 'Yes'); }); }); @@ -918,8 +864,15 @@ describe('aclRequired field in evaluateBucketPolicyWithIAM', () => { const request = { serverAccessLog: {} }; const bucket = makeBucket(ownerCanonicalId); const authInfo = makeAuthInfo(otherCanonicalId, 'arn:aws:iam::999999999999:user/other'); - evaluateBucketPolicyWithIAM(bucket, 'objectDelete', otherCanonicalId, authInfo, - { objectDelete: false }, stubLog, request); + evaluateBucketPolicyWithIAM( + bucket, + 'objectDelete', + otherCanonicalId, + authInfo, + { objectDelete: false }, + stubLog, + request, + ); assert.strictEqual(request.serverAccessLog.aclRequired, undefined); }); }); diff --git a/tests/unit/api/apiUtils/rateLimit/cache.js b/tests/unit/api/apiUtils/rateLimit/cache.js index cb3e705fa9..421fa6c9c0 100644 --- a/tests/unit/api/apiUtils/rateLimit/cache.js +++ b/tests/unit/api/apiUtils/rateLimit/cache.js @@ -29,13 +29,10 @@ describe('test limit config cache storage', () => { it('should add config to cache', () => { setCachedConfig(namespace.bucket, 'foo', 10, constants.rateLimitDefaultConfigCacheTTL); - assert.deepStrictEqual( - configCache.get(`${namespace.bucket}:foo`), - { - expiry: now + constants.rateLimitDefaultConfigCacheTTL, - value: 10, - } - ); + assert.deepStrictEqual(configCache.get(`${namespace.bucket}:foo`), { + expiry: now + constants.rateLimitDefaultConfigCacheTTL, + value: 10, + }); }); it('should get a non expired config', () => { diff --git a/tests/unit/api/apiUtils/rateLimit/cleanup.js b/tests/unit/api/apiUtils/rateLimit/cleanup.js index eab3e5901b..db2721c8a2 100644 --- a/tests/unit/api/apiUtils/rateLimit/cleanup.js +++ b/tests/unit/api/apiUtils/rateLimit/cleanup.js @@ -1,10 +1,7 @@ const assert = require('assert'); const sinon = require('sinon'); -const { - startCleanupJob, - stopCleanupJob, -} = require('../../../../../lib/api/apiUtils/rateLimit/cleanup'); +const { startCleanupJob, stopCleanupJob } = require('../../../../../lib/api/apiUtils/rateLimit/cleanup'); const constants = require('../../../../../constants'); describe('Rate limit cleanup job', () => { @@ -32,9 +29,11 @@ describe('Rate limit cleanup job', () => { startCleanupJob(mockLog, { skipUnref: true }); assert(mockLog.info.calledOnce); - assert(mockLog.info.calledWith('Starting rate limit cleanup job', { - interval: constants.rateLimitCleanupInterval, - })); + assert( + mockLog.info.calledWith('Starting rate limit cleanup job', { + interval: constants.rateLimitCleanupInterval, + }), + ); assert(setTimeoutSpy.calledOnce); assert.strictEqual(setTimeoutSpy.firstCall.args[1], constants.rateLimitCleanupInterval); }); diff --git a/tests/unit/api/apiUtils/rateLimit/config.js b/tests/unit/api/apiUtils/rateLimit/config.js index 170cde6118..bf0f84916e 100644 --- a/tests/unit/api/apiUtils/rateLimit/config.js +++ b/tests/unit/api/apiUtils/rateLimit/config.js @@ -404,7 +404,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond" must be of type object/, ); }); @@ -440,7 +439,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond.limit" must be larger than or equal to 0/, ); }); @@ -516,7 +514,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond.burstCapacity" must be larger than or equal to 0/, ); }); @@ -553,7 +550,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultConfig.requestsPerSecond.burstCapacity" must be a number/, ); }); @@ -603,7 +599,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"bucket.defaultBurstCapacity" must be larger than or equal to 0/, ); }); @@ -1066,7 +1061,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"account.defaultConfig.requestsPerSecond.burstCapacity" must be larger than or equal to 0/, ); }); @@ -1103,7 +1097,6 @@ describe('parseRateLimitConfig', () => { assert.throws( () => parseRateLimitConfig(config), - // eslint-disable-next-line max-len /rateLimiting configuration is invalid.*"account.defaultConfig.requestsPerSecond.burstCapacity" must be a number/, ); }); diff --git a/tests/unit/api/apiUtils/tagConditionKeys.js b/tests/unit/api/apiUtils/tagConditionKeys.js index 3a86628a7d..31ce684f93 100644 --- a/tests/unit/api/apiUtils/tagConditionKeys.js +++ b/tests/unit/api/apiUtils/tagConditionKeys.js @@ -8,8 +8,11 @@ const { TaggingConfigTester, createRequestContext, } = require('../../helpers'); -const { tagConditionKeyAuth, updateRequestContextsWithTags, makeTagQuery } = - require('../../../../lib/api/apiUtils/authorization/tagConditionKeys'); +const { + tagConditionKeyAuth, + updateRequestContextsWithTags, + makeTagQuery, +} = require('../../../../lib/api/apiUtils/authorization/tagConditionKeys'); const { bucketPut } = require('../../../../lib/api/bucketPut'); const objectPut = require('../../../../lib/api/objectPut'); @@ -29,21 +32,22 @@ const bucketPutReq = { const taggingUtil = new TaggingConfigTester(); -const objectPutReq = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-tagging': makeTagQuery(taggingUtil.getTags()), +const objectPutReq = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-tagging': makeTagQuery(taggingUtil.getTags()), + }, + url: `/${bucketName}/${objectKey}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectKey}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', -}, postBody); + postBody, +); -const objectPutRequestContexts = [ - createRequestContext('objectPut', objectPutReq), -]; +const objectPutRequestContexts = [createRequestContext('objectPut', objectPutReq)]; const objectGetReq = { bucketName, @@ -87,8 +91,7 @@ describe('updateRequestContextsWithTags', () => { updateRequestContextsWithTags(objectPutReq, objectPutRequestContexts, 'objectPut', log, err => { assert.ifError(err); assert(objectPutRequestContexts[0].getNeedTagEval()); - assert.strictEqual(objectPutRequestContexts[0].getRequestObjTags(), - makeTagQuery(taggingUtil.getTags())); + assert.strictEqual(objectPutRequestContexts[0].getRequestObjTags(), makeTagQuery(taggingUtil.getTags())); assert.strictEqual(objectPutRequestContexts[0].getExistingObjTag(), null); done(); }); @@ -97,14 +100,12 @@ describe('updateRequestContextsWithTags', () => { it('should update multiple request contexts with existing object tags', done => { objectPut(authInfo, objectPutReq, 'foobar', log, err => { assert.ifError(err); - updateRequestContextsWithTags(objectGetReq, objectGetRequestContexts, 'objectGet', log, - err => { + updateRequestContextsWithTags(objectGetReq, objectGetRequestContexts, 'objectGet', log, err => { assert.ifError(err); // FIXME introduced by CLDSRV-256, this syntax should be allowed by the linter for (const requestContext of objectGetRequestContexts) { assert(requestContext.getNeedTagEval()); - assert.strictEqual(requestContext.getExistingObjTag(), - makeTagQuery(taggingUtil.getTags())); + assert.strictEqual(requestContext.getExistingObjTag(), makeTagQuery(taggingUtil.getTags())); assert.strictEqual(requestContext.getRequestObjTags(), null); } done(); diff --git a/tests/unit/api/bucketACLauth.js b/tests/unit/api/bucketACLauth.js index 125fed71d5..0f104ba948 100644 --- a/tests/unit/api/bucketACLauth.js +++ b/tests/unit/api/bucketACLauth.js @@ -1,8 +1,7 @@ const assert = require('assert'); const BucketInfo = require('arsenal').models.BucketInfo; const constants = require('../../../constants'); -const { isBucketAuthorized } - = require('../../../lib/api/apiUtils/authorization/permissionChecks'); +const { isBucketAuthorized } = require('../../../lib/api/apiUtils/authorization/permissionChecks'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); const lifecycleServiceAccountId = '0123456789abcdef/lifecycle'; @@ -16,12 +15,10 @@ const ownerCanonicalId = authInfo.getCanonicalID(); const altAcctAuthInfo = makeAuthInfo(altAccessKey); const accountToVet = altAcctAuthInfo.getCanonicalID(); -const bucket = new BucketInfo('niftyBucket', ownerCanonicalId, - authInfo.getAccountDisplayName(), creationDate); +const bucket = new BucketInfo('niftyBucket', ownerCanonicalId, authInfo.getAccountDisplayName(), creationDate); const log = new DummyRequestLogger(); -describe('bucket authorization for bucketGet, bucketHead, ' + - 'objectGet, and objectHead', () => { +describe('bucket authorization for bucketGet, bucketHead, ' + 'objectGet, and objectHead', () => { // Reset the bucket ACLs afterEach(() => { bucket.setFullAcl({ @@ -44,70 +41,97 @@ describe('bucket authorization for bucketGet, bucketHead, ' + const orders = [ { - it: 'should allow access to bucket owner', canned: '', - id: ownerCanonicalId, response: trueArray, auth: authInfo, + it: 'should allow access to bucket owner', + canned: '', + id: ownerCanonicalId, + response: trueArray, + auth: authInfo, }, { - it: 'should allow access to user in bucket owner account', canned: '', - id: ownerCanonicalId, response: trueArray, auth: userAuthInfo, + it: 'should allow access to user in bucket owner account', + canned: '', + id: ownerCanonicalId, + response: trueArray, + auth: userAuthInfo, }, { it: 'should allow access to lifecycle service account', - canned: '', id: lifecycleServiceAccountId, response: trueArray, + canned: '', + id: lifecycleServiceAccountId, + response: trueArray, }, { - it: 'should allow public-user access for unknown ' + - 'service account and private canned ACL', - canned: '', id: unknownServiceAccountId, + it: 'should allow public-user access for unknown ' + 'service account and private canned ACL', + canned: '', + id: unknownServiceAccountId, response: falseArrayBucketTrueArrayObject, }, { it: 'should allow access to anyone if canned public-read ACL', - canned: 'public-read', id: accountToVet, response: trueArray, + canned: 'public-read', + id: accountToVet, + response: trueArray, auth: altAcctAuthInfo, }, { it: 'should allow access to anyone if canned public-read-write ACL', - canned: 'public-read-write', id: accountToVet, response: trueArray, + canned: 'public-read-write', + id: accountToVet, + response: trueArray, auth: altAcctAuthInfo, }, { - it: 'should not allow request on the bucket (bucketGet, bucketHead)' - + ' but should allow request on the object (objectGet, objectHead)' - + ' to public user if authenticated-read ACL', - canned: 'authenticated-read', id: constants.publicId, - response: falseArrayBucketTrueArrayObject, auth: altAcctAuthInfo, + it: + 'should not allow request on the bucket (bucketGet, bucketHead)' + + ' but should allow request on the object (objectGet, objectHead)' + + ' to public user if authenticated-read ACL', + canned: 'authenticated-read', + id: constants.publicId, + response: falseArrayBucketTrueArrayObject, + auth: altAcctAuthInfo, }, { - it: 'should allow access to any authenticated user if authenticated' - + '-read ACL', canned: 'authenticated-read', id: accountToVet, - response: trueArray, auth: altAcctAuthInfo, + it: 'should allow access to any authenticated user if authenticated' + '-read ACL', + canned: 'authenticated-read', + id: accountToVet, + response: trueArray, + auth: altAcctAuthInfo, }, { - it: 'should not allow request on the bucket (bucketGet, bucketHead)' - + ' but should allow request on the object (objectGet, objectHead)' - + ' to public user if private canned ACL', - canned: '', id: accountToVet, - response: falseArrayBucketTrueArrayObject, auth: altAcctAuthInfo, + it: + 'should not allow request on the bucket (bucketGet, bucketHead)' + + ' but should allow request on the object (objectGet, objectHead)' + + ' to public user if private canned ACL', + canned: '', + id: accountToVet, + response: falseArrayBucketTrueArrayObject, + auth: altAcctAuthInfo, }, { - it: 'should not allow request on the bucket (bucketGet, bucketHead)' - + ' but should allow request on the object (objectGet, objectHead)' - + ' to just any user if private canned ACL', - canned: '', id: accountToVet, - response: falseArrayBucketTrueArrayObject, auth: altAcctAuthInfo, + it: + 'should not allow request on the bucket (bucketGet, bucketHead)' + + ' but should allow request on the object (objectGet, objectHead)' + + ' to just any user if private canned ACL', + canned: '', + id: accountToVet, + response: falseArrayBucketTrueArrayObject, + auth: altAcctAuthInfo, }, { - it: 'should allow access to user if account was granted' - + ' FULL_CONTROL', - canned: '', id: accountToVet, response: trueArray, - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'should allow access to user if account was granted' + ' FULL_CONTROL', + canned: '', + id: accountToVet, + response: trueArray, + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { - it: 'should not allow access to just any user if private' - + ' canned ACL', - canned: '', id: accountToVet, response: trueArray, - aclParam: ['READ', accountToVet], auth: altAcctAuthInfo, + it: 'should not allow access to just any user if private' + ' canned ACL', + canned: '', + id: accountToVet, + response: trueArray, + aclParam: ['READ', accountToVet], + auth: altAcctAuthInfo, }, ]; @@ -117,8 +141,7 @@ describe('bucket authorization for bucketGet, bucketHead, ' + bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } bucket.setCannedAcl(value.canned); - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, value.id, value.auth, log)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, value.id, value.auth, log)); assert.deepStrictEqual(results, value.response); done(); }); @@ -139,45 +162,45 @@ describe('bucket authorization for bucketGetACL', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketGetACL', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketGetACL', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketGetACL', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketGetACL', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = [ { it: 'log group only if canned log-delivery-write acl', - id: constants.logId, canned: 'log-delivery-write', auth: null, + id: constants.logId, + canned: 'log-delivery-write', + auth: null, }, { it: 'account only if account was granted FULL_CONTROL right', - id: accountToVet, aclParam: ['FULL_CONTROL', accountToVet], + id: accountToVet, + aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, }, { it: 'account only if account was granted READ_ACP right', - id: accountToVet, aclParam: ['READ_ACP', accountToVet], + id: accountToVet, + aclParam: ['READ_ACP', accountToVet], auth: altAcctAuthInfo, }, ]; orders.forEach(value => { it(`should allow access to ${value.it}`, done => { - const noAuthResult = isBucketAuthorized(bucket, 'bucketGetACL', - value.id); + const noAuthResult = isBucketAuthorized(bucket, 'bucketGetACL', value.id); assert.strictEqual(noAuthResult, false); if (value.aclParam) { bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } else if (value.canned) { bucket.setCannedAcl(value.canned); } - const authorizedResult = isBucketAuthorized(bucket, 'bucketGetACL', - value.id, value.auth); + const authorizedResult = isBucketAuthorized(bucket, 'bucketGetACL', value.id, value.auth); assert.strictEqual(authorizedResult, true); done(); }); @@ -198,27 +221,22 @@ describe('bucket authorization for bucketPutACL', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketPutACL', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketPutACL', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketPutACL', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketPutACL', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = ['FULL_CONTROL', 'WRITE_ACP']; orders.forEach(value => { - it('should allow access to account if ' + - `account was granted ${value} right`, done => { - const noAuthResult = isBucketAuthorized(bucket, 'bucketPutACL', - accountToVet, altAcctAuthInfo); + it('should allow access to account if ' + `account was granted ${value} right`, done => { + const noAuthResult = isBucketAuthorized(bucket, 'bucketPutACL', accountToVet, altAcctAuthInfo); assert.strictEqual(noAuthResult, false); bucket.setSpecificAcl(accountToVet, value); - const authorizedResult = isBucketAuthorized(bucket, 'bucketPutACL', - accountToVet, altAcctAuthInfo); + const authorizedResult = isBucketAuthorized(bucket, 'bucketPutACL', accountToVet, altAcctAuthInfo); assert.strictEqual(authorizedResult, true); done(); }); @@ -239,26 +257,28 @@ describe('bucket authorization for bucketOwnerAction', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketDeleteCors', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketDeleteCors', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketDeleteCors', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketDeleteCors', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = [ { - it: 'other account (even if other account has FULL_CONTROL rights' - + ' in bucket)', id: accountToVet, canned: '', - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'other account (even if other account has FULL_CONTROL rights' + ' in bucket)', + id: accountToVet, + canned: '', + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { it: 'public user (even if bucket is public read write)', - id: constants.publicId, canned: 'public-read-write', auth: altAcctAuthInfo, + id: constants.publicId, + canned: 'public-read-write', + auth: altAcctAuthInfo, }, ]; orders.forEach(value => { @@ -267,8 +287,7 @@ describe('bucket authorization for bucketOwnerAction', () => { bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } bucket.setCannedAcl(value.canned); - const result = isBucketAuthorized(bucket, 'bucketDeleteCors', - value.id, value.auth); + const result = isBucketAuthorized(bucket, 'bucketDeleteCors', value.id, value.auth); assert.strictEqual(result, false); done(); }); @@ -289,26 +308,28 @@ describe('bucket authorization for bucketDelete', () => { }); it('should allow access to bucket owner', () => { - const result = isBucketAuthorized(bucket, 'bucketDelete', - ownerCanonicalId, authInfo); + const result = isBucketAuthorized(bucket, 'bucketDelete', ownerCanonicalId, authInfo); assert.strictEqual(result, true); }); it('should allow access to user in bucket owner account', () => { - const result = isBucketAuthorized(bucket, 'bucketDelete', - ownerCanonicalId, userAuthInfo); + const result = isBucketAuthorized(bucket, 'bucketDelete', ownerCanonicalId, userAuthInfo); assert.strictEqual(result, true); }); const orders = [ { - it: 'other account (even if other account has FULL_CONTROL rights ' - + 'in bucket)', id: accountToVet, canned: '', - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'other account (even if other account has FULL_CONTROL rights ' + 'in bucket)', + id: accountToVet, + canned: '', + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { it: 'public user (even if bucket is public read write)', - id: constants.publicId, canned: 'public-read-write', auth: null, + id: constants.publicId, + canned: 'public-read-write', + auth: null, }, ]; orders.forEach(value => { @@ -340,14 +361,12 @@ describe('bucket authorization for objectDelete and objectPut', () => { const requestTypes = ['objectDelete', 'objectPut']; it('should allow access to bucket owner', () => { - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, ownerCanonicalId, authInfo)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, ownerCanonicalId, authInfo)); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to user in bucket owner account', () => { - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, ownerCanonicalId, userAuthInfo)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, ownerCanonicalId, userAuthInfo)); assert.deepStrictEqual(results, [true, true]); }); @@ -355,45 +374,50 @@ describe('bucket authorization for objectDelete and objectPut', () => { // NOTE objectPut is not needed for lifecycle but still // allowed, we would want more fine-grained implementation of // ACLs for service accounts later. - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, lifecycleServiceAccountId)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, lifecycleServiceAccountId)); assert.deepStrictEqual(results, [true, true]); }); it('should deny access to unknown service account', () => { - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, unknownServiceAccountId)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, unknownServiceAccountId)); assert.deepStrictEqual(results, [false, false]); }); const orders = [ { it: 'anyone if canned public-read-write ACL', - canned: 'public-read-write', id: constants.publicId, + canned: 'public-read-write', + id: constants.publicId, response: [true, true], }, { - it: 'user if account was granted FULL_CONTROL', canned: '', - id: accountToVet, response: [false, false], - aclParam: ['FULL_CONTROL', accountToVet], auth: altAcctAuthInfo, + it: 'user if account was granted FULL_CONTROL', + canned: '', + id: accountToVet, + response: [false, false], + aclParam: ['FULL_CONTROL', accountToVet], + auth: altAcctAuthInfo, }, { - it: 'user if account was granted WRITE right', canned: '', - id: accountToVet, response: [false, false], - aclParam: ['WRITE', accountToVet], auth: altAcctAuthInfo, + it: 'user if account was granted WRITE right', + canned: '', + id: accountToVet, + response: [false, false], + aclParam: ['WRITE', accountToVet], + auth: altAcctAuthInfo, }, ]; orders.forEach(value => { it(`should allow access to ${value.it}`, done => { bucket.setCannedAcl(value.canned); - const noAuthResults = requestTypes.map(type => - isBucketAuthorized(bucket, type, value.id, value.auth)); + const noAuthResults = requestTypes.map(type => isBucketAuthorized(bucket, type, value.id, value.auth)); assert.deepStrictEqual(noAuthResults, value.response); if (value.aclParam) { bucket.setSpecificAcl(value.aclParam[1], value.aclParam[0]); } const authResults = requestTypes.map(type => - isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo)); + isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo), + ); assert.deepStrictEqual(authResults, [true, true]); done(); }); @@ -401,14 +425,11 @@ describe('bucket authorization for objectDelete and objectPut', () => { }); describe('bucket authorization for objectPutACL and objectGetACL', () => { - it('should allow access to anyone since checks ' + - 'are done at object level', done => { + it('should allow access to anyone since checks ' + 'are done at object level', done => { const requestTypes = ['objectPutACL', 'objectGetACL']; - const results = requestTypes.map(type => - isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo)); + const results = requestTypes.map(type => isBucketAuthorized(bucket, type, accountToVet, altAcctAuthInfo)); assert.deepStrictEqual(results, [true, true]); - const publicUserResults = requestTypes.map(type => - isBucketAuthorized(bucket, type, constants.publicId)); + const publicUserResults = requestTypes.map(type => isBucketAuthorized(bucket, type, constants.publicId)); assert.deepStrictEqual(publicUserResults, [true, true]); done(); }); diff --git a/tests/unit/api/bucketDelete.js b/tests/unit/api/bucketDelete.js index 6cd7d580ba..9442b8e64b 100644 --- a/tests/unit/api/bucketDelete.js +++ b/tests/unit/api/bucketDelete.js @@ -11,8 +11,7 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutEncryption = require('../../../lib/api/bucketPutEncryption'); const { templateSSEConfig, templateRequest } = require('../utils/bucketEncryption'); const constants = require('../../../constants'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const metadata = require('../metadataswitch'); const metadataMem = require('arsenal').storage.metadata.inMemory.metadata; const objectPut = require('../../../lib/api/objectPut'); @@ -20,7 +19,6 @@ const objectPutPart = require('../../../lib/api/objectPutPart'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const DummyRequest = require('../DummyRequest'); - const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; const authInfo = makeAuthInfo(canonicalID); @@ -32,55 +30,59 @@ const objectName = 'objectName'; const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; function createMPU(testRequest, initiateRequest, deleteOverviewMPUObj, cb) { - async.waterfall([ - next => bucketPut(authInfo, testRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), - (result, corsHeaders, next) => { - parseString(result, next); - }, - (json, next) => { - const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; - const md5Hash = crypto.createHash('md5'); - const bufferBody = Buffer.from(postBody); - md5Hash.update(bufferBody); - const calculatedHash = md5Hash.digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - objectKey: objectName, - namespace, - url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - query: { - partNumber: '1', - uploadId: testUploadId, - }, - calculatedHash, - }, postBody); - objectPutPart(authInfo, partRequest, undefined, log, err => { - if (err) { - return next(err); - } - return next(null, testUploadId); + async.waterfall( + [ + next => bucketPut(authInfo, testRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => { + parseString(result, next); + }, + (json, next) => { + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const md5Hash = crypto.createHash('md5'); + const bufferBody = Buffer.from(postBody); + md5Hash.update(bufferBody); + const calculatedHash = md5Hash.digest('hex'); + const partRequest = new DummyRequest( + { + bucketName, + objectKey: objectName, + namespace, + url: `/${objectName}?partNumber=1&uploadId=${testUploadId}`, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + calculatedHash, + }, + postBody, + ); + objectPutPart(authInfo, partRequest, undefined, log, err => { + if (err) { + return next(err); + } + return next(null, testUploadId); + }); + }, + ], + (err, testUploadId) => { + assert.strictEqual(err, null); + const mpuBucketKeyMap = metadataMem.metadata.keyMaps.get(mpuBucket); + assert.strictEqual(mpuBucketKeyMap.size, 2); + if (deleteOverviewMPUObj) { + const overviewKey = + `overview${constants.splitter}` + `${objectName}${constants.splitter}${testUploadId}`; + // remove overview key from in mem mpu bucket + mpuBucketKeyMap.delete(overviewKey); + assert.strictEqual(mpuBucketKeyMap.size, 1); + } + bucketDelete(authInfo, testRequest, log, err => { + assert.strictEqual(err, null); + cb(); }); }, - ], (err, testUploadId) => { - assert.strictEqual(err, null); - const mpuBucketKeyMap = - metadataMem.metadata.keyMaps.get(mpuBucket); - assert.strictEqual(mpuBucketKeyMap.size, 2); - if (deleteOverviewMPUObj) { - const overviewKey = `overview${constants.splitter}` + - `${objectName}${constants.splitter}${testUploadId}`; - // remove overview key from in mem mpu bucket - mpuBucketKeyMap.delete(overviewKey); - assert.strictEqual(mpuBucketKeyMap.size, 1); - } - bucketDelete(authInfo, testRequest, log, err => { - assert.strictEqual(err, null); - cb(); - }); - }); + ); } describe('bucketDelete API', () => { @@ -106,13 +108,16 @@ describe('bucketDelete API', () => { }; it('should return an error if the bucket is not empty', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName}`, - namespace, - objectKey: objectName, - }, postBody); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName}`, + namespace, + objectKey: objectName, + }, + postBody, + ); bucketPut(authInfo, testRequest, log, err => { assert.strictEqual(err, null); @@ -122,21 +127,22 @@ describe('bucketDelete API', () => { assert.strictEqual(err.is.BucketNotEmpty, true); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getName(), bucketName); - metadata.listObject(usersBucket, + metadata.listObject( + usersBucket, { prefix: authInfo.getCanonicalID() }, - log, (err, listResponse) => { - assert.strictEqual(listResponse.Contents.length, - 1); + log, + (err, listResponse) => { + assert.strictEqual(listResponse.Contents.length, 1); done(); - }); + }, + ); }); }); }); }); }); - it('should not return an error if the bucket has an initiated mpu', - done => { + it('should not return an error if the bucket has an initiated mpu', done => { bucketPut(authInfo, testRequest, log, err => { assert.strictEqual(err, null); initiateMultipartUpload(authInfo, initiateRequest, log, err => { @@ -155,23 +161,21 @@ describe('bucketDelete API', () => { metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(err.is.NoSuchBucket, true); assert.strictEqual(md, undefined); - metadata.listObject(usersBucket, { prefix: canonicalID }, - log, (err, listResponse) => { - assert.strictEqual(listResponse.Contents.length, 0); - done(); - }); + metadata.listObject(usersBucket, { prefix: canonicalID }, log, (err, listResponse) => { + assert.strictEqual(listResponse.Contents.length, 0); + done(); + }); }); }); }); }); - it('should delete a bucket even if the bucket has ongoing mpu', - done => createMPU(testRequest, initiateRequest, false, done)); + it('should delete a bucket even if the bucket has ongoing mpu', done => + createMPU(testRequest, initiateRequest, false, done)); // if only part object (and no overview objects) is in mpu shadow bucket - it('should delete a bucket even if the bucket has an orphan part', - done => createMPU(testRequest, initiateRequest, true, done)); - + it('should delete a bucket even if the bucket has an orphan part', done => + createMPU(testRequest, initiateRequest, true, done)); it('should prevent anonymous user delete bucket API access', done => { const publicAuthInfo = makeAuthInfo(constants.publicId); diff --git a/tests/unit/api/bucketDeleteCors.js b/tests/unit/api/bucketDeleteCors.js index 981fd62a8d..4a4f63aac2 100644 --- a/tests/unit/api/bucketDeleteCors.js +++ b/tests/unit/api/bucketDeleteCors.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); const bucketDeleteCors = require('../../../lib/api/bucketDeleteCors'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - CorsConfigTester } = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, CorsConfigTester } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -21,10 +18,8 @@ const testBucketPutRequest = { url: '/', actionImplicitDenies: false, }; -const testBucketPutCorsRequest = - corsUtil.createBucketCorsRequest('PUT', bucketName); -const testBucketDeleteCorsRequest = - corsUtil.createBucketCorsRequest('DELETE', bucketName); +const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName); +const testBucketDeleteCorsRequest = corsUtil.createBucketCorsRequest('DELETE', bucketName); describe('deleteBucketCors API', () => { beforeEach(done => { @@ -35,9 +30,8 @@ describe('deleteBucketCors API', () => { }); afterEach(() => cleanup()); - it('should delete a bucket\'s cors configuration in metadata', done => { - bucketDeleteCors(authInfo, testBucketDeleteCorsRequest, log, - err => { + it("should delete a bucket's cors configuration in metadata", done => { + bucketDeleteCors(authInfo, testBucketDeleteCorsRequest, log, err => { if (err) { process.stdout.write(`Unexpected err ${err}`); return done(err); diff --git a/tests/unit/api/bucketDeleteEncryption.js b/tests/unit/api/bucketDeleteEncryption.js index 13bf39bb6a..a8aa3c9f25 100644 --- a/tests/unit/api/bucketDeleteEncryption.js +++ b/tests/unit/api/bucketDeleteEncryption.js @@ -51,7 +51,8 @@ describe('bucketDeleteEncryption API', () => { }); }); }); - })); + }), + ); it('should remove sse and clear key for aws:kms with a configured master key id', done => { const post = templateSSEConfig({ algorithm: 'aws:kms', keyId: '12345' }); diff --git a/tests/unit/api/bucketDeleteLifecycle.js b/tests/unit/api/bucketDeleteLifecycle.js index 647ff09ba1..f7e5763a0e 100644 --- a/tests/unit/api/bucketDeleteLifecycle.js +++ b/tests/unit/api/bucketDeleteLifecycle.js @@ -4,10 +4,7 @@ const async = require('async'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutLifecycle = require('../../../lib/api/bucketDeleteLifecycle'); const bucketDeleteLifecycle = require('../../../lib/api/bucketDeleteLifecycle'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -22,12 +19,13 @@ function _makeRequest(includeXml) { actionImplicitDenies: false, }; if (includeXml) { - request.post = '' + - '' + - 'Enabled' + - '1' + - ''; + request.post = + '' + + '' + + 'Enabled' + + '1' + + ''; } return request; } @@ -44,17 +42,20 @@ describe('deleteBucketLifecycle API', () => { }); }); it('should delete bucket lifecycle', done => { - async.series([ - next => bucketPutLifecycle(authInfo, _makeRequest(true), log, next), - next => bucketDeleteLifecycle(authInfo, _makeRequest(), log, next), - // eslint-disable-next-line no-unused-vars - next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), - ], (err, results) => { - assert.equal(err, null, `Expected success, got error: ${err}`); - const bucket = results[2]; - const lifecycleConfig = bucket.getLifecycleConfiguration(); - assert.equal(lifecycleConfig, null); - done(); - }); + async.series( + [ + next => bucketPutLifecycle(authInfo, _makeRequest(true), log, next), + next => bucketDeleteLifecycle(authInfo, _makeRequest(), log, next), + // eslint-disable-next-line no-unused-vars + next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), + ], + (err, results) => { + assert.equal(err, null, `Expected success, got error: ${err}`); + const bucket = results[2]; + const lifecycleConfig = bucket.getLifecycleConfiguration(); + assert.equal(lifecycleConfig, null); + done(); + }, + ); }); }); diff --git a/tests/unit/api/bucketDeletePolicy.js b/tests/unit/api/bucketDeletePolicy.js index 69bf9234db..fc683bcecd 100644 --- a/tests/unit/api/bucketDeletePolicy.js +++ b/tests/unit/api/bucketDeletePolicy.js @@ -4,10 +4,7 @@ const async = require('async'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const bucketDeletePolicy = require('../../../lib/api/bucketDeletePolicy'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -50,17 +47,20 @@ describe('deleteBucketPolicy API', () => { }); }); it('should delete bucket policy', done => { - async.series([ - next => bucketPutPolicy(authInfo, _makeRequest(true), log, next), - next => bucketDeletePolicy(authInfo, _makeRequest(), log, next), - // eslint-disable-next-line no-unused-vars - next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), - ], (err, results) => { - assert.equal(err, null, `Expected success, got error: ${err}`); - const bucket = results[2]; - const bucketPolicy = bucket.getBucketPolicy(); - assert.equal(bucketPolicy, null); - done(); - }); + async.series( + [ + next => bucketPutPolicy(authInfo, _makeRequest(true), log, next), + next => bucketDeletePolicy(authInfo, _makeRequest(), log, next), + // eslint-disable-next-line no-unused-vars + next => metadata.getBucket(bucketName, log, (err, bucket, raftSessionId) => next(err, bucket)), + ], + (err, results) => { + assert.equal(err, null, `Expected success, got error: ${err}`); + const bucket = results[2]; + const bucketPolicy = bucket.getBucketPolicy(); + assert.equal(bucketPolicy, null); + done(); + }, + ); }); }); diff --git a/tests/unit/api/bucketDeleteTagging.js b/tests/unit/api/bucketDeleteTagging.js index 7eeb98f1f8..73f17bf539 100644 --- a/tests/unit/api/bucketDeleteTagging.js +++ b/tests/unit/api/bucketDeleteTagging.js @@ -1,11 +1,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const bucketPutTagging = require('../../../lib/api/bucketPutTagging'); const bucketGetTagging = require('../../../lib/api/bucketGetTagging'); const bucketDeleteTagging = require('../../../lib/api/bucketDeleteTagging'); @@ -30,45 +26,41 @@ describe('deleteBucketTagging API', () => { it('should delete tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.strictEqual(err, undefined); - const testBucketGetTaggingRequest = taggingUtil - .createBucketTaggingRequest('GET', bucketName); - return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, - (err, xml) => { + const testBucketGetTaggingRequest = taggingUtil.createBucketTaggingRequest('GET', bucketName); + return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, (err, xml) => { + assert.ifError(err); + assert.strictEqual(xml, taggingUtil.constructXml()); + const testBucketDeleteTaggingRequest = taggingUtil.createBucketTaggingRequest('DELETE', bucketName); + return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, log, err => { assert.ifError(err); - assert.strictEqual(xml, taggingUtil.constructXml()); - const testBucketDeleteTaggingRequest = taggingUtil - .createBucketTaggingRequest('DELETE', bucketName); - return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, - log, err => { - assert.ifError(err); - return bucketGetTagging(authInfo, testBucketGetTaggingRequest, - log, err => { - assert(err.NoSuchTagSet); - return done(); - }); - }); + return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, err => { + assert(err.NoSuchTagSet); + return done(); + }); }); + }); }); }); it('should return access denied if the authorization check fails', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.ifError(err); - const testBucketDeleteTaggingRequest = taggingUtil - .createBucketTaggingRequest('DELETE', bucketName, null, true); - return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, - log, err => { - assert(err.AccessDenied); - return done(); - }); + const testBucketDeleteTaggingRequest = taggingUtil.createBucketTaggingRequest( + 'DELETE', + bucketName, + null, + true, + ); + return bucketDeleteTagging(authInfo, testBucketDeleteTaggingRequest, log, err => { + assert(err.AccessDenied); + return done(); + }); }); }); }); diff --git a/tests/unit/api/bucketDeleteWebsite.js b/tests/unit/api/bucketDeleteWebsite.js index 41bd286c87..4d1623e363 100644 --- a/tests/unit/api/bucketDeleteWebsite.js +++ b/tests/unit/api/bucketDeleteWebsite.js @@ -3,19 +3,14 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketDeleteWebsite = require('../../../lib/api/bucketDeleteWebsite'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - WebsiteConfig } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, WebsiteConfig } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); const bucketName = 'bucketname'; const config = new WebsiteConfig('index.html', 'error.html'); -config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, -{ KeyPrefixEquals: 'docs/' }); +config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, { KeyPrefixEquals: 'docs/' }); const testBucketPutRequest = { bucketName, headers: { host: `${bucketName}.s3.amazonaws.com` }, @@ -31,8 +26,7 @@ const testBucketDeleteWebsiteRequest = { query: { website: '' }, actionImplicitDenies: false, }; -const testBucketPutWebsiteRequest = Object.assign({ post: config.getXml() }, - testBucketDeleteWebsiteRequest); +const testBucketPutWebsiteRequest = Object.assign({ post: config.getXml() }, testBucketDeleteWebsiteRequest); describe('deleteBucketWebsite API', () => { beforeEach(done => { @@ -43,9 +37,8 @@ describe('deleteBucketWebsite API', () => { }); afterEach(() => cleanup()); - it('should delete a bucket\'s website configuration in metadata', done => { - bucketDeleteWebsite(authInfo, testBucketDeleteWebsiteRequest, log, - err => { + it("should delete a bucket's website configuration in metadata", done => { + bucketDeleteWebsite(authInfo, testBucketDeleteWebsiteRequest, log, err => { if (err) { process.stdout.write(`Unexpected err ${err}`); return done(err); @@ -55,8 +48,7 @@ describe('deleteBucketWebsite API', () => { process.stdout.write(`Err retrieving bucket MD ${err}`); return done(err); } - assert.strictEqual(bucket.getWebsiteConfiguration(), - null); + assert.strictEqual(bucket.getWebsiteConfiguration(), null); return done(); }); }); diff --git a/tests/unit/api/bucketGet.js b/tests/unit/api/bucketGet.js index 7968ca78a1..61fcbc4565 100644 --- a/tests/unit/api/bucketGet.js +++ b/tests/unit/api/bucketGet.js @@ -24,40 +24,55 @@ const objectName1 = `${prefix}${delimiter}objectName1`; const objectName2 = `${prefix}${delimiter}objectName2`; const objectName3 = 'invalidURI~~~b'; const objectName4 = `${objectName1}&><"\'`; -const testPutBucketRequest = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}`, - namespace, -}, Buffer.alloc(0)); -const testPutObjectRequest1 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName1}`, - namespace, - objectKey: objectName1, -}, postBody); -const testPutObjectRequest2 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName2}`, - namespace, - objectKey: objectName2, -}, postBody); -const testPutObjectRequest3 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName3}`, - namespace, - objectKey: objectName3, -}, postBody); -const testPutObjectRequest4 = new DummyRequest({ - bucketName, - headers: {}, - url: `/${bucketName}/${objectName3}`, - namespace, - objectKey: objectName4, -}, postBody); +const testPutBucketRequest = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}`, + namespace, + }, + Buffer.alloc(0), +); +const testPutObjectRequest1 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName1}`, + namespace, + objectKey: objectName1, + }, + postBody, +); +const testPutObjectRequest2 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName2}`, + namespace, + objectKey: objectName2, + }, + postBody, +); +const testPutObjectRequest3 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName3}`, + namespace, + objectKey: objectName3, + }, + postBody, +); +const testPutObjectRequest4 = new DummyRequest( + { + bucketName, + headers: {}, + url: `/${bucketName}/${objectName3}`, + namespace, + objectKey: objectName4, + }, + postBody, +); const baseGetRequest = { bucketName, @@ -72,39 +87,33 @@ const tests = [ name: 'list of all objects if no delimiter specified', request: Object.assign({ query: {}, url: baseUrl }, baseGetRequest), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], - objectName1); - assert.strictEqual(result.ListBucketResult.Contents[2].Key[0], - objectName2); - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], objectName1); + assert.strictEqual(result.ListBucketResult.Contents[2].Key[0], objectName2); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); }, }, { - name: 'return name of common prefix of common prefix objects if ' + - 'delimiter and prefix specified', - request: Object.assign({ - url: `/${bucketName}?delimiter=${delimiter}&prefix=${prefix}`, - query: { delimiter, prefix }, - }, baseGetRequest), + name: 'return name of common prefix of common prefix objects if ' + 'delimiter and prefix specified', + request: Object.assign( + { + url: `/${bucketName}?delimiter=${delimiter}&prefix=${prefix}`, + query: { delimiter, prefix }, + }, + baseGetRequest, + ), assertion: result => - assert.strictEqual(result.ListBucketResult - .CommonPrefixes[0].Prefix[0], `${prefix}${delimiter}`), + assert.strictEqual(result.ListBucketResult.CommonPrefixes[0].Prefix[0], `${prefix}${delimiter}`), }, { name: 'return empty list when max-keys is set to 0', - request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.Contents, undefined), + request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.Contents, undefined), }, { name: 'return no more keys than max-keys specified', - request: Object.assign({ query: { 'max-keys': '1' }, url: baseUrl }, - baseGetRequest), + request: Object.assign({ query: { 'max-keys': '1' }, url: baseUrl }, baseGetRequest), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); assert.strictEqual(result.ListBucketResult.Contents[1], undefined); }, }, @@ -115,16 +124,12 @@ const tests = [ query: { 'max-keys': '1' }, url: baseUrl, }, - baseGetRequest + baseGetRequest, ), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); assert.strictEqual(result.ListBucketResult.Contents[1], undefined); - assert.strictEqual( - result.ListBucketResult.NextContinuationToken[0], - 'aW52YWxpZFVSSX5+fmI=' - ); + assert.strictEqual(result.ListBucketResult.NextContinuationToken[0], 'aW52YWxpZFVSSX5+fmI='); }, }, { @@ -134,43 +139,30 @@ const tests = [ query: { 'encoding-type': 'url', 'max-keys': '1' }, url: baseUrl, }, - baseGetRequest + baseGetRequest, ), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - objectName3); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], objectName3); assert.strictEqual(result.ListBucketResult.Contents[1], undefined); - assert.strictEqual( - result.ListBucketResult.NextContinuationToken[0], - 'aW52YWxpZFVSSX5+fmI=' - ); + assert.strictEqual(result.ListBucketResult.NextContinuationToken[0], 'aW52YWxpZFVSSX5+fmI='); }, }, { - name: 'return max-keys number from request even if greater than ' + - 'actual keys returned', - request: Object.assign({ query: { 'max-keys': '99999' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.MaxKeys[0], '99999'), + name: 'return max-keys number from request even if greater than ' + 'actual keys returned', + request: Object.assign({ query: { 'max-keys': '99999' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.MaxKeys[0], '99999'), }, { name: 'return max-keys number from request even when value is 0', - request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), + request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), }, { name: 'url encode object key name if requested', - request: Object.assign( - { query: { 'encoding-type': 'url' }, url: baseUrl }, - baseGetRequest), + request: Object.assign({ query: { 'encoding-type': 'url' }, url: baseUrl }, baseGetRequest), assertion: result => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - querystring.escape(objectName3)); - assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], - querystring.escape(objectName1)); + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], querystring.escape(objectName3)); + assert.strictEqual(result.ListBucketResult.Contents[1].Key[0], querystring.escape(objectName1)); }, }, ]; @@ -184,28 +176,25 @@ describe('bucketGet API', () => { it(`should ${test.name}`, done => { const testGetRequest = test.request; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest1, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest2, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest3, undefined, log, next), - (_, next) => - bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - test.assertion(result); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest1, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest2, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest3, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + test.assertion(result); + done(); + }, + ); }); }); it('should return an InvalidArgument error if max-keys == -1', done => { - const testGetRequest = Object.assign({ query: { 'max-keys': '-1' } }, - baseGetRequest); + const testGetRequest = Object.assign({ query: { 'max-keys': '-1' } }, baseGetRequest); bucketGet(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.InvalidArgument, true); done(); @@ -213,81 +202,73 @@ describe('bucketGet API', () => { }); it('should escape invalid xml characters in object key names', done => { - const testGetRequest = Object.assign({ query: {}, url: baseUrl }, - baseGetRequest); - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, testPutObjectRequest4, - undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, - log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], - testPutObjectRequest4.objectKey); - done(); - }); + const testGetRequest = Object.assign({ query: {}, url: baseUrl }, baseGetRequest); + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest4, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + assert.strictEqual(result.ListBucketResult.Contents[0].Key[0], testPutObjectRequest4.objectKey); + done(); + }, + ); }); it('should return xml that refers to the s3 docs for xml specs', done => { - const testGetRequest = Object.assign({ query: {}, url: baseUrl }, - baseGetRequest); - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => - bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - assert.strictEqual(result.ListBucketResult.$.xmlns, - 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + const testGetRequest = Object.assign({ query: {}, url: baseUrl }, baseGetRequest); + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); }); -const testsForV2 = [...tests, +const testsForV2 = [ + ...tests, { name: 'return no owner info when --fetch-owner option is not used', request: Object.assign({ query: {}, url: baseUrl }, baseGetRequest), assertion: result => { - const owners - = result.ListBucketResult.Contents.filter(c => c.Owner); + const owners = result.ListBucketResult.Contents.filter(c => c.Owner); assert.strictEqual(owners.length, 0); }, }, { name: 'return owner info when --fetch-owner option is used', - request: Object.assign({ query: { 'fetch-owner': 'true' }, - url: baseUrl }, baseGetRequest), + request: Object.assign({ query: { 'fetch-owner': 'true' }, url: baseUrl }, baseGetRequest), assertion: result => { - const owners - = result.ListBucketResult.Contents.filter(c => + const owners = result.ListBucketResult.Contents.filter( + c => c.Owner[0].ID[0] === authInfo.canonicalID && - c.Owner[0].DisplayName[0] === authInfo.accountDisplayName); - assert.strictEqual(owners.length, - result.ListBucketResult.Contents.length); + c.Owner[0].DisplayName[0] === authInfo.accountDisplayName, + ); + assert.strictEqual(owners.length, result.ListBucketResult.Contents.length); }, }, { name: 'return no owner info when --no-fetch-owner option is used', - request: Object.assign({ query: { 'fetch-owner': 'false' }, - url: baseUrl }, baseGetRequest), + request: Object.assign({ query: { 'fetch-owner': 'false' }, url: baseUrl }, baseGetRequest), assertion: result => { - const owners - = result.ListBucketResult.Contents.filter(c => c.Owner); + const owners = result.ListBucketResult.Contents.filter(c => c.Owner); assert.strictEqual(owners.length, 0); }, }, { name: 'return max-keys number from request even when value is 0', - request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, - baseGetRequest), - assertion: result => - assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), + request: Object.assign({ query: { 'max-keys': '0' }, url: baseUrl }, baseGetRequest), + assertion: result => assert.strictEqual(result.ListBucketResult.MaxKeys[0], '0'), }, ]; @@ -299,286 +280,350 @@ describe('bucketGet API V2', () => { testsForV2.forEach(test => { /* eslint-disable no-param-reassign */ test.request.query['list-type'] = 2; - test.request.url = test.request.url.indexOf('?') > -1 ? - `${test.request.url}&list-type=2` : - `${test.request.url}?list-type=2`; + test.request.url = + test.request.url.indexOf('?') > -1 ? `${test.request.url}&list-type=2` : `${test.request.url}?list-type=2`; /* eslint-enable no-param-reassign */ it(`should ${test.name}`, done => { const testGetRequest = test.request; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest1, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest2, undefined, log, next), - (_, next) => objectPut(authInfo, - testPutObjectRequest3, undefined, log, next), - (_, next) => - bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (_, result) => { - // v2 requests should return 'KeyCount' in response - const keyCount = - Number.parseInt(result.ListBucketResult.KeyCount[0], 10); - const keysReturned = result.ListBucketResult.Contents ? - result.ListBucketResult.Contents.length : 0; - assert.strictEqual(keyCount, keysReturned); - // assert the results from tests - test.assertion(result); - if (result.ListBucketResult.IsTruncated && result.ListBucketResult.IsTruncated[0] === 'false') { - assert.strictEqual(result.ListBucketResult.NextContinuationToken, undefined); - } - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest1, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest2, undefined, log, next), + (_, next) => objectPut(authInfo, testPutObjectRequest3, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (_, result) => { + // v2 requests should return 'KeyCount' in response + const keyCount = Number.parseInt(result.ListBucketResult.KeyCount[0], 10); + const keysReturned = result.ListBucketResult.Contents ? result.ListBucketResult.Contents.length : 0; + assert.strictEqual(keyCount, keysReturned); + // assert the results from tests + test.assertion(result); + if (result.ListBucketResult.IsTruncated && result.ListBucketResult.IsTruncated[0] === 'false') { + assert.strictEqual(result.ListBucketResult.NextContinuationToken, undefined); + } + done(); + }, + ); }); }); describe('x-amz-optional-object-attributes header', () => { it('should return an error if the header is empty', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = ''; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }, + ); }); it('should return an error for invalid optional attributes', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'InvalidAttribute'; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketGet(authInfo, testGetRequest, log, next), - ], err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketGet(authInfo, testGetRequest, log, next), + ], + err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }, + ); }); it('should accept wildcard value', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-*'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(err, null); - assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); it('should return an error for a mix of valid and invalid attributes', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,InvalidAttribute'; - async.series([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - next => bucketGet(authInfo, testGetRequest, log, next), - ], err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + async.series( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + next => bucketGet(authInfo, testGetRequest, log, next), + ], + err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }, + ); }); it('should handle attributes with leading/trailing whitespace', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = ' x-amz-meta-foo '; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); it('should handle multiple valid attributes', done => { - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,x-amz-meta-foo,x-amz-meta-bar'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + assert.strictEqual(result.ListBucketResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); + }, + ); }); it('should return user metadata if requested and present', done => { const objectNameMeta = 'objectWithMeta'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-color'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content['x-amz-meta-color'][0], 'red'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + done(); + }, + ); }); it('should not duplicate elements when the header repeats tokens', done => { const objectNameMeta = 'objectWithRepeatedTokens'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,RestoreStatus,x-amz-meta-color,x-amz-meta-color'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content.RestoreStatus.length, 1); - assert.strictEqual(content['x-amz-meta-color'].length, 1); - assert.strictEqual(content['x-amz-meta-color'][0], 'red'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content.RestoreStatus.length, 1); + assert.strictEqual(content['x-amz-meta-color'].length, 1); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + done(); + }, + ); }); it('should return all user metadata if wildcard requested', done => { const objectNameMeta = 'objectWithMetaWildcard'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red', 'x-amz-meta-size': 'large' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red', 'x-amz-meta-size': 'large' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-*'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content['x-amz-meta-color'][0], 'red'); - assert.strictEqual(content['x-amz-meta-size'][0], 'large'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'red'); + assert.strictEqual(content['x-amz-meta-size'][0], 'large'); + done(); + }, + ); }); it('should return user metadata in version listing if requested', done => { const objectNameMeta = 'objectWithMetaVersion'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-ver': '1' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-ver': '1' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: { versions: '' }, - url: `${baseUrl}?versions`, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: { versions: '' }, + url: `${baseUrl}?versions`, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-ver'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const version = result.ListVersionsResult.Version[0]; - assert.strictEqual(version.Key[0], objectNameMeta); - assert.strictEqual(version['x-amz-meta-ver'][0], '1'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const version = result.ListVersionsResult.Version[0]; + assert.strictEqual(version.Key[0], objectNameMeta); + assert.strictEqual(version['x-amz-meta-ver'][0], '1'); + done(); + }, + ); }); it('should not include optional attributes on delete markers in versions listing', done => { const objectNameMeta = 'objectWithMetaAndDeleteMarker'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'red' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'red' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); const deleteRequest = new DummyRequest({ bucketName, headers: {}, @@ -593,70 +638,83 @@ describe('bucketGet API V2', () => { url: '/?versioning', query: { versioning: '' }, actionImplicitDenies: false, - post: '' - + 'Enabled', + post: + '' + + 'Enabled', }; - const testGetRequest = Object.assign({ - query: { versions: '' }, - url: `${baseUrl}?versions`, - }, baseGetRequest); - testGetRequest.headers['x-amz-optional-object-attributes'] = - 'RestoreStatus,x-amz-meta-color'; - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => bucketPutVersioning(authInfo, versioningRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => objectDelete(authInfo, deleteRequest, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const { Version, DeleteMarker } = result.ListVersionsResult; - assert.strictEqual(Version.length, 1); - assert.strictEqual(Version[0].Key[0], objectNameMeta); - assert.strictEqual(Version[0]['x-amz-meta-color'][0], 'red'); - assert.strictEqual(Version[0].RestoreStatus.length, 1); - - assert.strictEqual(DeleteMarker.length, 1); - assert.strictEqual(DeleteMarker[0].Key[0], objectNameMeta); - assert.strictEqual(DeleteMarker[0]['x-amz-meta-color'], undefined); - assert.strictEqual(DeleteMarker[0].RestoreStatus, undefined); - done(); - }); + const testGetRequest = Object.assign( + { + query: { versions: '' }, + url: `${baseUrl}?versions`, + }, + baseGetRequest, + ); + testGetRequest.headers['x-amz-optional-object-attributes'] = 'RestoreStatus,x-amz-meta-color'; + + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => bucketPutVersioning(authInfo, versioningRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => objectDelete(authInfo, deleteRequest, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const { Version, DeleteMarker } = result.ListVersionsResult; + assert.strictEqual(Version.length, 1); + assert.strictEqual(Version[0].Key[0], objectNameMeta); + assert.strictEqual(Version[0]['x-amz-meta-color'][0], 'red'); + assert.strictEqual(Version[0].RestoreStatus.length, 1); + + assert.strictEqual(DeleteMarker.length, 1); + assert.strictEqual(DeleteMarker[0].Key[0], objectNameMeta); + assert.strictEqual(DeleteMarker[0]['x-amz-meta-color'], undefined); + assert.strictEqual(DeleteMarker[0].RestoreStatus, undefined); + done(); + }, + ); }); it('should return user metadata as case insentive (lowercase header)', done => { const objectNameMeta = 'objectWithMeta'; - const putRequest = new DummyRequest({ - bucketName, - headers: { 'x-amz-meta-color': 'yellow' }, - url: `/${bucketName}/${objectNameMeta}`, - namespace, - objectKey: objectNameMeta, - }, postBody); + const putRequest = new DummyRequest( + { + bucketName, + headers: { 'x-amz-meta-color': 'yellow' }, + url: `/${bucketName}/${objectNameMeta}`, + namespace, + objectKey: objectNameMeta, + }, + postBody, + ); - const testGetRequest = Object.assign({ - query: {}, - url: baseUrl, - }, baseGetRequest); + const testGetRequest = Object.assign( + { + query: {}, + url: baseUrl, + }, + baseGetRequest, + ); testGetRequest.headers['x-amz-optional-object-attributes'] = 'x-amz-meta-coLor'; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (_, next) => objectPut(authInfo, putRequest, undefined, log, next), - (_, next) => bucketGet(authInfo, testGetRequest, log, next), - (result, _, next) => parseString(result, next), - ], - (err, result) => { - assert.strictEqual(err, null); - const content = result.ListBucketResult.Contents[0]; - assert.strictEqual(content.Key[0], objectNameMeta); - assert.strictEqual(content['x-amz-meta-color'][0], 'yellow'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (_, next) => objectPut(authInfo, putRequest, undefined, log, next), + (_, next) => bucketGet(authInfo, testGetRequest, log, next), + (result, _, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual(err, null); + const content = result.ListBucketResult.Contents[0]; + assert.strictEqual(content.Key[0], objectNameMeta); + assert.strictEqual(content['x-amz-meta-color'][0], 'yellow'); + done(); + }, + ); }); }); }); diff --git a/tests/unit/api/bucketGetACL.js b/tests/unit/api/bucketGetACL.js index 5a4327a9d7..3f013982a9 100644 --- a/tests/unit/api/bucketGetACL.js +++ b/tests/unit/api/bucketGetACL.js @@ -41,7 +41,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'private', }, url: '/?acl', @@ -49,24 +49,26 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, - testGetACLRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1], undefined); + done(); + }, + ); }); it('should get a canned public-read-write ACL', done => { @@ -74,7 +76,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read-write', }, url: '/?acl', @@ -82,35 +84,36 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0].URI[0], - constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0].URI[0], - constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2] - .Permission[0], 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3], undefined); + done(); + }, + ); }); it('should get a canned public-read ACL', done => { @@ -118,7 +121,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read', }, url: '/?acl', @@ -126,29 +129,31 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0].URI[0], - constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned authenticated-read ACL', done => { @@ -156,7 +161,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'authenticated-read', }, url: '/?acl', @@ -164,30 +169,31 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.allAuthedUsersId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.allAuthedUsersId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned log-delivery-write ACL', done => { @@ -195,7 +201,7 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'log-delivery-write', }, url: '/?acl', @@ -203,36 +209,36 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.logId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1] - .Permission[0], 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .URI[0], constants.logId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2] - .Permission[0], 'READ_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.logId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'WRITE'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].URI[0], + constants.logId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'READ_ACP'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3], undefined); + done(); + }, + ); }); it('should get specifically set ACLs', done => { @@ -240,104 +246,96 @@ describe('bucketGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: '/?acl', query: { acl: '' }, actionImplicitDenies: false, }; - const canonicalIDforSample1 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; - const canonicalIDforSample2 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; + const canonicalIDforSample1 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; + const canonicalIDforSample2 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalIDforSample1); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], canonicalIDforSample2); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .ID[0], canonicalIDforSample2); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Permission[0], - 'WRITE_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .ID[0], canonicalIDforSample1); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Permission[0], - 'READ_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4] - .Permission[0], 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[5].Grantee[0] - .URI[0], constants.logId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[5] - .Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[6], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalIDforSample1, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + canonicalIDforSample2, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].ID[0], + canonicalIDforSample2, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].ID[0], + canonicalIDforSample1, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3].Permission[0], 'READ_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[4].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[4].Permission[0], 'WRITE'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[5].Grantee[0].URI[0], + constants.logId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[5].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[6], undefined); + done(); + }, + ); }); - const grantsByURI = [ - constants.publicId, - constants.allAuthedUsersId, - constants.logId, - ]; + const grantsByURI = [constants.publicId, constants.allAuthedUsersId, constants.logId]; grantsByURI.forEach(uri => { - it('should get all ACLs when predefined group - ' + - `${uri} is used for multiple grants`, done => { + it('should get all ACLs when predefined group - ' + `${uri} is used for multiple grants`, done => { const testPutACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': `uri = ${uri}`, 'x-amz-grant-read': `uri = ${uri}`, 'x-amz-grant-write': `uri = ${uri}`, @@ -349,36 +347,35 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, - log, next), (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, - testGetACLRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.ifError(err); - const grants = - result.AccessControlPolicy.AccessControlList[0].Grant; - grants.forEach(grant => { - assert.strictEqual(grant.Permission.length, 1); - assert.strictEqual(grant.Grantee.length, 1); - assert.strictEqual(grant.Grantee[0].URI.length, 1); - assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); - }); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.ifError(err); + const grants = result.AccessControlPolicy.AccessControlList[0].Grant; + grants.forEach(grant => { + assert.strictEqual(grant.Permission.length, 1); + assert.strictEqual(grant.Grantee.length, 1); + assert.strictEqual(grant.Grantee[0].URI.length, 1); + assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); + }); + done(); + }, + ); }); }); - it('should get all ACLs when predefined groups are used for ' + - 'more than one grant', done => { + it('should get all ACLs when predefined groups are used for ' + 'more than one grant', done => { const { allAuthedUsersId, publicId } = constants; const testPutACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-write': `uri = ${allAuthedUsersId} `, 'x-amz-grant-write-acp': `uri = ${allAuthedUsersId} `, 'x-amz-grant-read': `uri = ${publicId} `, @@ -389,33 +386,33 @@ describe('bucketGetACL API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - bucketPutACL(authInfo, testPutACLRequest, log, next), - (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, - log, next), - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.ifError(err); - const grants = - result.AccessControlPolicy.AccessControlList[0].Grant; - grants.forEach(grant => { - const permissions = grant.Permission; - assert.strictEqual(permissions.length, 1); - const permission = permissions[0]; - assert.strictEqual(grant.Grantee.length, 1); - const grantees = grant.Grantee[0].URI; - assert.strictEqual(grantees.length, 1); - const grantee = grantees[0]; - if (['WRITE', 'WRITE_ACP'].includes(permission)) { - assert.strictEqual(grantee, constants.allAuthedUsersId); - } - if (['READ', 'READ_ACP'].includes(permission)) { - assert.strictEqual(grantee, constants.publicId); - } - }); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => bucketPutACL(authInfo, testPutACLRequest, log, next), + (corsHeaders, next) => bucketGetACL(authInfo, testGetACLRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.ifError(err); + const grants = result.AccessControlPolicy.AccessControlList[0].Grant; + grants.forEach(grant => { + const permissions = grant.Permission; + assert.strictEqual(permissions.length, 1); + const permission = permissions[0]; + assert.strictEqual(grant.Grantee.length, 1); + const grantees = grant.Grantee[0].URI; + assert.strictEqual(grantees.length, 1); + const grantee = grantees[0]; + if (['WRITE', 'WRITE_ACP'].includes(permission)) { + assert.strictEqual(grantee, constants.allAuthedUsersId); + } + if (['READ', 'READ_ACP'].includes(permission)) { + assert.strictEqual(grantee, constants.publicId); + } + }); + done(); + }, + ); }); }); diff --git a/tests/unit/api/bucketGetCors.js b/tests/unit/api/bucketGetCors.js index 01c8bf1839..675125f45c 100644 --- a/tests/unit/api/bucketGetCors.js +++ b/tests/unit/api/bucketGetCors.js @@ -4,10 +4,7 @@ const crypto = require('crypto'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); const bucketGetCors = require('../../../lib/api/bucketGetCors'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -32,25 +29,24 @@ function _makeCorsRequest(xml) { if (xml) { request.post = xml; - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } const testGetCorsRequest = _makeCorsRequest(); function _comparePutGetXml(sampleXml, done) { - const fullXml = '' + - `${sampleXml}`; + const fullXml = + '' + + `${sampleXml}`; const testPutCorsRequest = _makeCorsRequest(fullXml); bucketPutCors(authInfo, testPutCorsRequest, log, err => { if (err) { process.stdout.write(`Err putting cors config ${err}`); return done(err); } - return bucketGetCors(authInfo, testGetCorsRequest, log, - (err, res) => { + return bucketGetCors(authInfo, testGetCorsRequest, log, (err, res) => { assert.strictEqual(err, null, `Unexpected err ${err}`); assert.strictEqual(res, fullXml); done(); @@ -65,8 +61,7 @@ describe('getBucketCors API', () => { }); afterEach(() => cleanup()); - it('should return same XML as uploaded for AllowedMethod and ' + - 'AllowedOrigin', done => { + it('should return same XML as uploaded for AllowedMethod and ' + 'AllowedOrigin', done => { const sampleXml = '' + 'PUT' + @@ -91,7 +86,7 @@ describe('getBucketCors API', () => { _comparePutGetXml(sampleXml, done); }); - it('should return same XML as uploaded for AllowedHeader\'s', done => { + it("should return same XML as uploaded for AllowedHeader's", done => { const sampleXml = '' + 'PUT' + @@ -103,7 +98,7 @@ describe('getBucketCors API', () => { _comparePutGetXml(sampleXml, done); }); - it('should return same XML as uploaded for ExposedHeader\'s', done => { + it("should return same XML as uploaded for ExposedHeader's", done => { const sampleXml = '' + 'PUT' + diff --git a/tests/unit/api/bucketGetLifecycle.js b/tests/unit/api/bucketGetLifecycle.js index 95025d74a7..d80d336b72 100644 --- a/tests/unit/api/bucketGetLifecycle.js +++ b/tests/unit/api/bucketGetLifecycle.js @@ -3,12 +3,8 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetLifecycle = require('../../../lib/api/bucketGetLifecycle'); const bucketPutLifecycle = require('../../../lib/api/bucketPutLifecycle'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); -const { getLifecycleRequest, getLifecycleXml } = - require('../utils/lifecycleHelpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const { getLifecycleRequest, getLifecycleXml } = require('../utils/lifecycleHelpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -25,8 +21,7 @@ describe('getBucketLifecycle API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); afterEach(() => cleanup()); - it('should return NoSuchLifecycleConfiguration error if ' + - 'bucket has no lifecycle', done => { + it('should return NoSuchLifecycleConfiguration error if ' + 'bucket has no lifecycle', done => { const lifecycleRequest = getLifecycleRequest(bucketName); bucketGetLifecycle(authInfo, lifecycleRequest, log, err => { assert.strictEqual(err.is.NoSuchLifecycleConfiguration, true); @@ -36,8 +31,7 @@ describe('getBucketLifecycle API', () => { describe('after bucket lifecycle has been put', () => { beforeEach(done => { - const putRequest = - getLifecycleRequest(bucketName, getLifecycleXml()); + const putRequest = getLifecycleRequest(bucketName, getLifecycleXml()); bucketPutLifecycle(authInfo, putRequest, log, err => { assert.equal(err, null); done(); @@ -48,8 +42,7 @@ describe('getBucketLifecycle API', () => { const getRequest = getLifecycleRequest(bucketName); bucketGetLifecycle(authInfo, getRequest, log, (err, res) => { assert.equal(err, null); - const expectedXML = '' + - `${getLifecycleXml()}`; + const expectedXML = '' + `${getLifecycleXml()}`; assert.deepStrictEqual(expectedXML, res); done(); }); diff --git a/tests/unit/api/bucketGetLocation.js b/tests/unit/api/bucketGetLocation.js index 27f0815263..46ef4e2dc5 100644 --- a/tests/unit/api/bucketGetLocation.js +++ b/tests/unit/api/bucketGetLocation.js @@ -2,16 +2,10 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetLocation = require('../../../lib/api/bucketGetLocation'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const { config } = require('../../../lib/Config'); -const { - LOCATION_NAME_DMF, - LOCATION_NAME_CRR, -} = require('../../constants'); +const { LOCATION_NAME_DMF, LOCATION_NAME_CRR } = require('../../constants'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -37,11 +31,13 @@ const testGetLocationRequest = { const locationConstraints = config.locationConstraints; function getBucketRequestObject(location) { - const post = location ? '' + - '' + - `${location}` + - '' : undefined; + const post = location + ? '' + + '' + + `${location}` + + '' + : undefined; return Object.assign({ post }, testBucketPutRequest); } @@ -64,13 +60,11 @@ describe('getBucketLocation API', () => { }); afterEach(() => cleanup()); it(`should return ${location} LocationConstraint xml`, done => { - bucketGetLocation(authInfo, testGetLocationRequest, log, - (err, res) => { - assert.strictEqual(err, null, - `Unexpected ${err} getting location constraint`); - const xml = ` - ` + - `${location}`; + bucketGetLocation(authInfo, testGetLocationRequest, log, (err, res) => { + assert.strictEqual(err, null, `Unexpected ${err} getting location constraint`); + const xml = + ` + ` + `${location}`; assert.deepStrictEqual(res, xml); return done(); }); @@ -86,13 +80,11 @@ describe('getBucketLocation API', () => { }); afterEach(() => cleanup()); it('should return empty string LocationConstraint xml', done => { - bucketGetLocation(authInfo, testGetLocationRequest, log, - (err, res) => { - assert.strictEqual(err, null, - `Unexpected ${err} getting location constraint`); - const xml = ` - ` + - ''; + bucketGetLocation(authInfo, testGetLocationRequest, log, (err, res) => { + assert.strictEqual(err, null, `Unexpected ${err} getting location constraint`); + const xml = + ` + ` + ''; assert.deepStrictEqual(res, xml); return done(); }); diff --git a/tests/unit/api/bucketGetLogging.js b/tests/unit/api/bucketGetLogging.js index d6bb09a950..acce545928 100644 --- a/tests/unit/api/bucketGetLogging.js +++ b/tests/unit/api/bucketGetLogging.js @@ -61,13 +61,15 @@ function createGetLoggingRequest(bucketName, headers = {}) { } function createValidLoggingXML(targetBucket, targetPrefix = 'logs/') { - return '' + + return ( + '' + '' + '' + `${targetBucket}` + `${targetPrefix}` + '' + - ''; + '' + ); } describe('bucketGetLogging API', () => { @@ -90,7 +92,8 @@ describe('bucketGetLogging API', () => { assert.ifError(err); assert(xml); // Should return empty BucketLoggingStatus - const expectedXML = '\n' + + const expectedXML = + '\n' + ''; assert.strictEqual(xml, expectedXML); done(); @@ -163,7 +166,8 @@ describe('bucketGetLogging API', () => { assert.ifError(err); // Disable logging - const disableXML = '' + + const disableXML = + '' + ''; const disableRequest = createLoggingRequest(bucketName, disableXML); diff --git a/tests/unit/api/bucketGetNotification.js b/tests/unit/api/bucketGetNotification.js index 5091b45ef9..f48606d411 100644 --- a/tests/unit/api/bucketGetNotification.js +++ b/tests/unit/api/bucketGetNotification.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetNotification = require('../../../lib/api/bucketGetNotification'); const bucketPutNotification = require('../../../lib/api/bucketPutNotification'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -40,7 +37,8 @@ function getNotificationXml() { const filterName = 'Prefix'; const filterValue = 'logs/'; - return '' + + return ( + '' + '' + `${id}` + `${queueArn}` + @@ -51,10 +49,10 @@ function getNotificationXml() { `${filterValue}` + '' + '' + - ''; + '' + ); } - describe('getBucketNotification API', () => { before(cleanup); beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); @@ -70,8 +68,7 @@ describe('getBucketNotification API', () => { describe('after bucket notification has been put', () => { beforeEach(done => { - const putRequest = - getNotificationRequest(bucketName, getNotificationXml()); + const putRequest = getNotificationRequest(bucketName, getNotificationXml()); bucketPutNotification(authInfo, putRequest, log, err => { assert.ifError(err); done(); @@ -82,8 +79,7 @@ describe('getBucketNotification API', () => { const getRequest = getNotificationRequest(bucketName); bucketGetNotification(authInfo, getRequest, log, (err, res) => { assert.ifError(err); - const expectedXML = '' + - `${getNotificationXml()}`; + const expectedXML = '' + `${getNotificationXml()}`; assert.deepStrictEqual(expectedXML, res); done(); }); diff --git a/tests/unit/api/bucketGetObjectLock.js b/tests/unit/api/bucketGetObjectLock.js index 39c55574e7..e5553da33b 100644 --- a/tests/unit/api/bucketGetObjectLock.js +++ b/tests/unit/api/bucketGetObjectLock.js @@ -20,7 +20,7 @@ const bucketPutReq = { const testBucketPutReqWithObjLock = { bucketName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'True', }, url: '/', @@ -31,7 +31,7 @@ function getObjectLockConfigRequest(bucketName, xml) { const request = { bucketName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }, url: '/?object-lock', @@ -59,10 +59,7 @@ function getObjectLockXml(mode, type, time) { // object lock is enabled and object lock configuration is set if (arguments.length === 3) { - xmlStr += xml.ruleOpen + - retentionMode + - retentionTime + - xml.ruleClose; + xmlStr += xml.ruleOpen + retentionMode + retentionTime + xml.ruleClose; } xmlStr += xml.objLockConfigClose; return xmlStr; @@ -72,14 +69,16 @@ describe('bucketGetObjectLock API', () => { before(done => bucketPut(authInfo, bucketPutReq, log, done)); after(cleanup); - it('should return ObjectLockConfigurationNotFoundError error if ' + - 'object lock is not enabled on the bucket', done => { - const objectLockRequest = getObjectLockConfigRequest(bucketName); - bucketGetObjectLock(authInfo, objectLockRequest, log, err => { - assert.strictEqual(err.is.ObjectLockConfigurationNotFoundError, true); - done(); - }); - }); + it( + 'should return ObjectLockConfigurationNotFoundError error if ' + 'object lock is not enabled on the bucket', + done => { + const objectLockRequest = getObjectLockConfigRequest(bucketName); + bucketGetObjectLock(authInfo, objectLockRequest, log, err => { + assert.strictEqual(err.is.ObjectLockConfigurationNotFoundError, true); + done(); + }); + }, + ); }); describe('bucketGetObjectLock API', () => { @@ -87,8 +86,7 @@ describe('bucketGetObjectLock API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutReqWithObjLock, log, done)); afterEach(cleanup); - it('should return config without \'rule\' if object lock configuration ' + - 'not set on the bucket', done => { + it("should return config without 'rule' if object lock configuration " + 'not set on the bucket', done => { const objectLockRequest = getObjectLockConfigRequest(bucketName); bucketGetObjectLock(authInfo, objectLockRequest, log, (err, res) => { assert.ifError(err); diff --git a/tests/unit/api/bucketGetPolicy.js b/tests/unit/api/bucketGetPolicy.js index 504f702c92..b51e378390 100644 --- a/tests/unit/api/bucketGetPolicy.js +++ b/tests/unit/api/bucketGetPolicy.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketGetPolicy = require('../../../lib/api/bucketGetPolicy'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -43,8 +40,7 @@ describe('getBucketPolicy API', () => { beforeEach(done => bucketPut(authInfo, testBasicRequest, log, done)); afterEach(() => cleanup()); - it('should return NoSuchBucketPolicy error if ' + - 'bucket has no policy', done => { + it('should return NoSuchBucketPolicy error if ' + 'bucket has no policy', done => { bucketGetPolicy(authInfo, testBasicRequest, log, err => { assert.strictEqual(err.is.NoSuchBucketPolicy, true); done(); diff --git a/tests/unit/api/bucketGetRateLimit.js b/tests/unit/api/bucketGetRateLimit.js index 2f65447795..e43625b62f 100644 --- a/tests/unit/api/bucketGetRateLimit.js +++ b/tests/unit/api/bucketGetRateLimit.js @@ -85,8 +85,7 @@ describe('bucketGetRateLimit API', () => { }); }); - it('should return NoSuchRateLimitConfig error if bucket exists but ' + - 'rate limit config is not set', done => { + it('should return NoSuchRateLimitConfig error if bucket exists but ' + 'rate limit config is not set', done => { bucketPut(regularAuthInfo, bucketPutReq, log, err => { assert.ifError(err); const rateLimitRequest = getRateLimitConfigRequest(bucketName); diff --git a/tests/unit/api/bucketGetReplication.js b/tests/unit/api/bucketGetReplication.js index 268c902025..be29801cf2 100644 --- a/tests/unit/api/bucketGetReplication.js +++ b/tests/unit/api/bucketGetReplication.js @@ -2,8 +2,7 @@ const assert = require('assert'); const { parseString } = require('xml2js'); const { DummyRequestLogger } = require('../helpers'); -const { getReplicationConfigurationXML } = - require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); +const { getReplicationConfigurationXML } = require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); // Compare the values from the parsedXML with the original configuration values. function checkXML(parsedXML, config) { @@ -58,29 +57,25 @@ describe("'getReplicationConfigurationXML' function", () => { it('should return XML from the bucket replication configuration', done => getAndCheckXML(getReplicationConfig(), done)); - it('should not return XML with StorageClass tag if `storageClass` ' + - 'property is omitted', done => { + it('should not return XML with StorageClass tag if `storageClass` ' + 'property is omitted', done => { const config = getReplicationConfig(); delete config.rules[0].storageClass; return getAndCheckXML(config, done); }); - it("should return XML with StorageClass tag set to 'Disabled' if " + - '`enabled` property is false', done => { + it("should return XML with StorageClass tag set to 'Disabled' if " + '`enabled` property is false', done => { const config = getReplicationConfig(); config.rules[0].enabled = false; return getAndCheckXML(config, done); }); - it('should return XML with a self-closing Prefix tag if `prefix` ' + - "property is ''", done => { + it('should return XML with a self-closing Prefix tag if `prefix` ' + "property is ''", done => { const config = getReplicationConfig(); config.rules[0].prefix = ''; return getAndCheckXML(config, done); }); - it('should return XML from the bucket replication configuration with ' + - 'multiple rules', done => { + it('should return XML from the bucket replication configuration with ' + 'multiple rules', done => { const config = getReplicationConfig(); config.rules.push({ id: 'test-id-2', diff --git a/tests/unit/api/bucketGetTagging.js b/tests/unit/api/bucketGetTagging.js index a1c5f502db..686d92d243 100644 --- a/tests/unit/api/bucketGetTagging.js +++ b/tests/unit/api/bucketGetTagging.js @@ -1,11 +1,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester, -} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const bucketPutTagging = require('../../../lib/api/bucketPutTagging'); const bucketGetTagging = require('../../../lib/api/bucketGetTagging'); const log = new DummyRequestLogger(); @@ -29,14 +25,11 @@ describe('getBucketTagging API', () => { it('should return tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.strictEqual(err, undefined); - const testBucketGetTaggingRequest = taggingUtil - .createBucketTaggingRequest('GET', bucketName); - return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, - (err, xml) => { + const testBucketGetTaggingRequest = taggingUtil.createBucketTaggingRequest('GET', bucketName); + return bucketGetTagging(authInfo, testBucketGetTaggingRequest, log, (err, xml) => { if (err) { process.stdout.write(`Err getting object tagging ${err}`); return done(err); @@ -49,19 +42,15 @@ describe('getBucketTagging API', () => { it('should return access denied if the authorization check fails', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert.strictEqual(err, undefined); - const testBucketGetTaggingRequest = taggingUtil - .createBucketTaggingRequest('GET', bucketName, true); + const testBucketGetTaggingRequest = taggingUtil.createBucketTaggingRequest('GET', bucketName, true); const badAuthInfo = makeAuthInfo('accessKey2'); - return bucketGetTagging(badAuthInfo, testBucketGetTaggingRequest, log, - err => { + return bucketGetTagging(badAuthInfo, testBucketGetTaggingRequest, log, err => { assert.strictEqual(err.AccessDenied, true); return done(); }); }); }); }); - diff --git a/tests/unit/api/bucketGetWebsite.js b/tests/unit/api/bucketGetWebsite.js index 75caf14129..eb3c9fc999 100644 --- a/tests/unit/api/bucketGetWebsite.js +++ b/tests/unit/api/bucketGetWebsite.js @@ -3,10 +3,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketGetWebsite = require('../../../lib/api/bucketGetWebsite'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } -= require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -37,18 +34,18 @@ function _makeWebsiteRequest(xml) { const testGetWebsiteRequest = _makeWebsiteRequest(); function _comparePutGetXml(sampleXml, done) { - const fullXml = '' + - `${sampleXml}`; + const fullXml = + '' + + `${sampleXml}`; const testPutWebsiteRequest = _makeWebsiteRequest(fullXml); bucketPutWebsite(authInfo, testPutWebsiteRequest, log, err => { if (err) { process.stdout.write(`Err putting website config ${err}`); return done(err); } - return bucketGetWebsite(authInfo, testGetWebsiteRequest, log, - (err, res) => { + return bucketGetWebsite(authInfo, testGetWebsiteRequest, log, (err, res) => { assert.strictEqual(err, null, `Unexpected err ${err}`); assert.strictEqual(res, fullXml); done(); @@ -64,8 +61,7 @@ describe('getBucketWebsite API', () => { afterEach(() => cleanup()); it('should return same IndexDocument XML as uploaded', done => { - const sampleXml = - 'index.html'; + const sampleXml = 'index.html'; _comparePutGetXml(sampleXml, done); }); it('should return same ErrorDocument XML as uploaded', done => { diff --git a/tests/unit/api/bucketHead.js b/tests/unit/api/bucketHead.js index 3c006a549c..b2fa311b30 100644 --- a/tests/unit/api/bucketHead.js +++ b/tests/unit/api/bucketHead.js @@ -37,8 +37,7 @@ describe('bucketHead API', () => { }); }); - it('should return no error if bucket exists and user is authorized', - done => { + it('should return no error if bucket exists and user is authorized', done => { bucketPut(authInfo, testRequest, log, () => { bucketHead(authInfo, testRequest, log, err => { assert.strictEqual(err, null); diff --git a/tests/unit/api/bucketPolicyAuth.js b/tests/unit/api/bucketPolicyAuth.js index 618f7b8908..67f4a3297b 100644 --- a/tests/unit/api/bucketPolicyAuth.js +++ b/tests/unit/api/bucketPolicyAuth.js @@ -2,8 +2,11 @@ const assert = require('assert'); const { BucketInfo, BucketPolicy } = require('arsenal').models; const AuthInfo = require('arsenal').auth.AuthInfo; const constants = require('../../../constants'); -const { isBucketAuthorized, isObjAuthorized, validatePolicyResource } - = require('../../../lib/api/apiUtils/authorization/permissionChecks'); +const { + isBucketAuthorized, + isObjAuthorized, + validatePolicyResource, +} = require('../../../lib/api/apiUtils/authorization/permissionChecks'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); const DummyRequest = require('../DummyRequest'); @@ -20,8 +23,12 @@ const altAcctCanonicalId = altAcctAuthInfo.getCanonicalID(); const accountId = authInfo.getShortid(); const altAcctId = altAcctAuthInfo.getShortid(); const creationDate = new Date().toJSON(); -const bucket = new BucketInfo('policyBucketAuthTester', bucketOwnerCanonicalId, - authInfo.getAccountDisplayName(), creationDate); +const bucket = new BucketInfo( + 'policyBucketAuthTester', + bucketOwnerCanonicalId, + authInfo.getAccountDisplayName(), + creationDate, +); const object = { 'owner-id': objectOwnerCanonicalId }; const bucAction = 'bucketHead'; const objAction = 'objectPut'; @@ -90,8 +97,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access if account id principal is contained in ' + - 'user arn of non-', + name: 'should allow access if account id principal is contained in ' + 'user arn of non-', bucketId: objectOwnerCanonicalId, bucketAuthInfo: user1AuthInfo, objectId: objectOwnerCanonicalId, @@ -103,8 +109,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access if account id principal is contained in ' + - 'account arn of non-', + name: 'should allow access if account id principal is contained in ' + 'account arn of non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -116,8 +121,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access if account arn principal is contained in ' + - 'user arn of non-', + name: 'should allow access if account arn principal is contained in ' + 'user arn of non-', bucketId: objectOwnerCanonicalId, bucketAuthInfo: user1AuthInfo, objectId: objectOwnerCanonicalId, @@ -129,8 +133,7 @@ const authTests = [ expected: true, }, { - name: 'should allow access even if user arn principal doesn\'t match ' + - 'user arn of user in account of ', + name: "should allow access even if user arn principal doesn't match " + 'user arn of user in account of ', bucketId: objectOwnerCanonicalId, bucketAuthInfo: user1AuthInfo, objectId: objectOwnerCanonicalId, @@ -142,8 +145,7 @@ const authTests = [ expected: true, }, { - name: 'should deny access if account arn principal doesn\'t match ' + - 'user arn of non-', + name: "should deny access if account arn principal doesn't match " + 'user arn of non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctUserAuthInfo, objectId: altAcctCanonicalId, @@ -155,8 +157,7 @@ const authTests = [ expected: false, }, { - name: 'should deny access if user arn principal doesn\'t match ' + - 'user arn of non-', + name: "should deny access if user arn principal doesn't match " + 'user arn of non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctUserAuthInfo, objectId: altAcctCanonicalId, @@ -168,7 +169,7 @@ const authTests = [ expected: false, }, { - name: 'should deny access if principal doesn\'t match non-', + name: "should deny access if principal doesn't match non-", bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -180,8 +181,7 @@ const authTests = [ expected: false, }, { - name: 'should allow access if principal and action match policy for ' + - 'non-', + name: 'should allow access if principal and action match policy for ' + 'non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -193,8 +193,7 @@ const authTests = [ expected: true, }, { - name: 'should deny access if principal matches but action does not ' + - 'match policy for non-', + name: 'should deny access if principal matches but action does not ' + 'match policy for non-', bucketId: altAcctCanonicalId, bucketAuthInfo: altAcctAuthInfo, objectId: altAcctCanonicalId, @@ -253,8 +252,7 @@ const resourceTests = [ expected: false, }, { - name: 'false if policy resource is array and any elements do not ' + - 'match bucket arn', + name: 'false if policy resource is array and any elements do not ' + 'match bucket arn', rValue: [`arn:aws:s3:::${bucketName}`, 'arn:aws:s3:::nomatch'], expected: false, }, @@ -263,50 +261,41 @@ const resourceTests = [ describe('bucket policy authorization', () => { describe('isBucketAuthorized with no policy set', () => { it('should allow access to bucket owner', done => { - const allowed = isBucketAuthorized(bucket, 'bucketPut', - bucketOwnerCanonicalId, null, log); + const allowed = isBucketAuthorized(bucket, 'bucketPut', bucketOwnerCanonicalId, null, log); assert.equal(allowed, true); done(); }); - it('should deny access to non-bucket owner', - done => { - const allowed = isBucketAuthorized(bucket, 'bucketPut', - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); + it('should deny access to non-bucket owner', done => { + const allowed = isBucketAuthorized(bucket, 'bucketPut', altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }); }); describe('isBucketAuthorized with bucket policy set', () => { beforeEach(function beFn() { - this.currentTest.basePolicy = new BucketPolicy(JSON.stringify( - basePolicyObj)).getBucketPolicy(); + this.currentTest.basePolicy = new BucketPolicy(JSON.stringify(basePolicyObj)).getBucketPolicy(); bucket.setBucketPolicy(this.currentTest.basePolicy); }); - it('should allow access to non-bucket owner if principal is set to "*"', - done => { - const allowed = isBucketAuthorized(bucket, bucAction, - altAcctCanonicalId, null, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to non-bucket owner if principal is set to "*"', done => { + const allowed = isBucketAuthorized(bucket, bucAction, altAcctCanonicalId, null, log); + assert.equal(allowed, true); + done(); + }); - it('should allow access to public user if principal is set to "*"', - done => { - const allowed = isBucketAuthorized(bucket, bucAction, - constants.publicId, publicUserAuthInfo, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to public user if principal is set to "*"', done => { + const allowed = isBucketAuthorized(bucket, bucAction, constants.publicId, publicUserAuthInfo, log); + assert.equal(allowed, true); + done(); + }); it('should deny access to public user if principal is not set to "*"', function itFn(done) { const newPolicy = this.test.basePolicy; newPolicy.Statement[0].Principal = { AWS: authInfo.getArn() }; bucket.setBucketPolicy(newPolicy); - const allowed = isBucketAuthorized(bucket, bucAction, - constants.publicId, publicUserAuthInfo, log); + const allowed = isBucketAuthorized(bucket, bucAction, constants.publicId, publicUserAuthInfo, log); assert.equal(allowed, false); done(); }); @@ -316,37 +305,35 @@ describe('bucket policy authorization', () => { const newPolicy = this.test.basePolicy; newPolicy.Statement[0][t.keyToChange] = t.bucketValue; bucket.setBucketPolicy(newPolicy); - const allowed = isBucketAuthorized(bucket, bucAction, - t.bucketId, t.bucketAuthInfo, log); + const allowed = isBucketAuthorized(bucket, bucAction, t.bucketId, t.bucketAuthInfo, log); assert.equal(allowed, t.expected); done(); }); }); - it('should deny access to non-bucket owner if two statements apply ' + - 'to principal but one denies access', function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[1] = { - Effect: 'Deny', - Principal: { CanonicalUser: [altAcctCanonicalId] }, - Resource: `arn:aws:s3:::${bucket.getName()}`, - Action: 's3:*', - }; - bucket.setBucketPolicy(newPolicy); - const allowed = isBucketAuthorized(bucket, bucAction, - altAcctCanonicalId, null, log); + it( + 'should deny access to non-bucket owner if two statements apply ' + 'to principal but one denies access', + function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[1] = { + Effect: 'Deny', + Principal: { CanonicalUser: [altAcctCanonicalId] }, + Resource: `arn:aws:s3:::${bucket.getName()}`, + Action: 's3:*', + }; + bucket.setBucketPolicy(newPolicy); + const allowed = isBucketAuthorized(bucket, bucAction, altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }, + ); + + it('should deny access to non-bucket owner with an unsupported action type', done => { + const allowed = isBucketAuthorized(bucket, 'unsupportedAction', altAcctCanonicalId, null, log); assert.equal(allowed, false); done(); }); - it('should deny access to non-bucket owner with an unsupported action type', - done => { - const allowed = isBucketAuthorized(bucket, 'unsupportedAction', - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); - it('should bypass bucket policy when request.bypassUserBucketPolicies is true', function () { // Create a request with the bypassUserBucketPolicies flag initially not set const request = { @@ -365,14 +352,12 @@ describe('bucket policy authorization', () => { bucket.setBucketPolicy(newPolicy); // Check that the policy denies access, as expected - assert.ok(!isBucketAuthorized(bucket, bucAction, - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok(!isBucketAuthorized(bucket, bucAction, bucketOwnerCanonicalId, user1AuthInfo, log, request)); // But with bypassUserBucketPolicies set to true, it should still be authorized // based on ACL permissions (which we mock to return true) request.bypassUserBucketPolicies = true; - assert.ok(isBucketAuthorized(bucket, bucAction, - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok(isBucketAuthorized(bucket, bucAction, bucketOwnerCanonicalId, user1AuthInfo, log, request)); }); }); @@ -382,79 +367,74 @@ describe('bucket policy authorization', () => { }); it('should allow access to object owner', done => { - const allowed = isObjAuthorized(bucket, object, objAction, - objectOwnerCanonicalId, null, log); + const allowed = isObjAuthorized(bucket, object, objAction, objectOwnerCanonicalId, null, log); assert.equal(allowed, true); done(); }); - it('should deny access to non-object owner', - done => { - const allowed = isObjAuthorized(bucket, object, objAction, - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); + it('should deny access to non-object owner', done => { + const allowed = isObjAuthorized(bucket, object, objAction, altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }); }); describe('isObjAuthorized with bucket policy set', () => { beforeEach(function beFn() { const newPolicyObj = basePolicyObj; - newPolicyObj.Statement.Resource = - `arn:aws:s3:::${bucket.getName()}/*`; - this.currentTest.basePolicy = new BucketPolicy(JSON.stringify( - newPolicyObj)).getBucketPolicy(); + newPolicyObj.Statement.Resource = `arn:aws:s3:::${bucket.getName()}/*`; + this.currentTest.basePolicy = new BucketPolicy(JSON.stringify(newPolicyObj)).getBucketPolicy(); bucket.setBucketPolicy(this.currentTest.basePolicy); }); - it('should allow access to non-object owner if principal is set to "*"', - done => { - const allowed = isObjAuthorized(bucket, object, objAction, - altAcctCanonicalId, null, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to non-object owner if principal is set to "*"', done => { + const allowed = isObjAuthorized(bucket, object, objAction, altAcctCanonicalId, null, log); + assert.equal(allowed, true); + done(); + }); - it('should allow access to public user if principal is set to "*"', - done => { - const allowed = isObjAuthorized(bucket, object, objAction, - constants.publicId, publicUserAuthInfo, log); - assert.equal(allowed, true); - done(); - }); + it('should allow access to public user if principal is set to "*"', done => { + const allowed = isObjAuthorized(bucket, object, objAction, constants.publicId, publicUserAuthInfo, log); + assert.equal(allowed, true); + done(); + }); authTests.forEach(t => { it(`${t.name}object owner`, function itFn(done) { const newPolicy = this.test.basePolicy; newPolicy.Statement[0][t.keyToChange] = t.objectValue; bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, objAction, - t.objectId, t.objectAuthInfo, log, null, t.impDenies); + const allowed = isObjAuthorized( + bucket, + object, + objAction, + t.objectId, + t.objectAuthInfo, + log, + null, + t.impDenies, + ); assert.equal(allowed, t.expected); done(); }); }); - it('should allow access to non-object owner for objectHead action with s3:GetObject permission', - function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[0].Action = ['s3:GetObject']; - bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, 'objectHead', - altAcctCanonicalId, altAcctAuthInfo, log); - assert.equal(allowed, true); - done(); - }); - it('should deny access to non-object owner for objectHead action without s3:GetObject permission', - function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[0].Action = ['s3:PutObject']; - bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, 'objectHead', - altAcctCanonicalId, altAcctAuthInfo, log); - assert.equal(allowed, false); - done(); - }); + it('should allow access to non-object owner for objectHead action with s3:GetObject permission', function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[0].Action = ['s3:GetObject']; + bucket.setBucketPolicy(newPolicy); + const allowed = isObjAuthorized(bucket, object, 'objectHead', altAcctCanonicalId, altAcctAuthInfo, log); + assert.equal(allowed, true); + done(); + }); + it('should deny access to non-object owner for objectHead action without s3:GetObject permission', function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[0].Action = ['s3:PutObject']; + bucket.setBucketPolicy(newPolicy); + const allowed = isObjAuthorized(bucket, object, 'objectHead', altAcctCanonicalId, altAcctAuthInfo, log); + assert.equal(allowed, false); + done(); + }); it('should bypass bucket policy when request.bypassUserBucketPolicies is true', function () { // Create a request with the bypassUserBucketPolicies flag initially not set @@ -474,40 +454,41 @@ describe('bucket policy authorization', () => { bucket.setBucketPolicy(newPolicy); // Check that the policy denies access, as expected - assert.ok(!isObjAuthorized(bucket, object, 'objectGet', - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok( + !isObjAuthorized(bucket, object, 'objectGet', bucketOwnerCanonicalId, user1AuthInfo, log, request), + ); // But with bypassUserBucketPolicies set to true, it should still be authorized // based on ACL permissions (which we mock to return true) request.bypassUserBucketPolicies = true; - assert.ok(isObjAuthorized(bucket, object, 'objectGet', - bucketOwnerCanonicalId, user1AuthInfo, log, request)); + assert.ok( + isObjAuthorized(bucket, object, 'objectGet', bucketOwnerCanonicalId, user1AuthInfo, log, request), + ); }); - it('should deny access to non-object owner if two statements apply ' + - 'to principal but one denies access', function itFn(done) { - const newPolicy = this.test.basePolicy; - newPolicy.Statement[1] = { - Effect: 'Deny', - Principal: { CanonicalUser: [altAcctCanonicalId] }, - Resource: `arn:aws:s3:::${bucket.getName()}/*`, - Action: 's3:*', - }; - bucket.setBucketPolicy(newPolicy); - const allowed = isObjAuthorized(bucket, object, objAction, - altAcctCanonicalId, null, log); + it( + 'should deny access to non-object owner if two statements apply ' + 'to principal but one denies access', + function itFn(done) { + const newPolicy = this.test.basePolicy; + newPolicy.Statement[1] = { + Effect: 'Deny', + Principal: { CanonicalUser: [altAcctCanonicalId] }, + Resource: `arn:aws:s3:::${bucket.getName()}/*`, + Action: 's3:*', + }; + bucket.setBucketPolicy(newPolicy); + const allowed = isObjAuthorized(bucket, object, objAction, altAcctCanonicalId, null, log); + assert.equal(allowed, false); + done(); + }, + ); + + it('should deny access to non-object owner with an unsupported action type', done => { + const allowed = isObjAuthorized(bucket, object, 'unsupportedAction', altAcctCanonicalId, null, log); assert.equal(allowed, false); done(); }); - it('should deny access to non-object owner with an unsupported action type', - done => { - const allowed = isObjAuthorized(bucket, object, 'unsupportedAction', - altAcctCanonicalId, null, log); - assert.equal(allowed, false); - done(); - }); - it('should allow access when implicitDeny true with Allow bucket policy', function itFn() { const requestTypes = ['objectPut', 'objectDelete']; const impDenies = { @@ -523,8 +504,16 @@ describe('bucket policy authorization', () => { try { const results = requestTypes.map(type => { - const allowed = isObjAuthorized(bucket, object, type, - altAcctCanonicalId, altAcctAuthInfo, log, null, impDenies); + const allowed = isObjAuthorized( + bucket, + object, + type, + altAcctCanonicalId, + altAcctAuthInfo, + log, + null, + impDenies, + ); return allowed; }); assert.deepStrictEqual(results, [true, true]); @@ -550,8 +539,16 @@ describe('bucket policy authorization', () => { bucket.setBucketPolicy(newPolicy); const results = requestTypes.map(type => { - const allowed = isObjAuthorized(bucket, object, type, - altAcctCanonicalId, altAcctAuthInfo, log, null, impDenies); + const allowed = isObjAuthorized( + bucket, + object, + type, + altAcctCanonicalId, + altAcctAuthInfo, + log, + null, + impDenies, + ); return allowed; }); assert.deepStrictEqual(results, [false, false]); @@ -564,14 +561,12 @@ describe('bucket policy authorization', () => { const newPolicy = basePolicyObj; newPolicy.Statement.Resource = t.rValue; newPolicy.Statement = [newPolicy.Statement]; - assert.equal( - validatePolicyResource(bucketName, newPolicy), t.expected); + assert.equal(validatePolicyResource(bucketName, newPolicy), t.expected); done(); }); }); - it('should return false if any statement resource does not match ' + - 'bucket arn', done => { + it('should return false if any statement resource does not match ' + 'bucket arn', done => { const newPolicy = basePolicyObj; newPolicy.Statement = [newPolicy.Statement]; newPolicy.Statement[1] = basePolicyObj.Statement; @@ -591,23 +586,15 @@ describe('bucket policy authorization', () => { CanonicalUser: [altAcctCanonicalId], }, Action: 's3:*', - Resource: [ - `arn:aws:s3:::${bucket.getName()}`, - `arn:aws:s3:::${bucket.getName()}/*`, - ], + Resource: [`arn:aws:s3:::${bucket.getName()}`, `arn:aws:s3:::${bucket.getName()}/*`], }, { Effect: 'Deny', Principal: { CanonicalUser: [altAcctCanonicalId], }, - Action: [ - 's3:PutObjectRetention', - ], - Resource: [ - `arn:aws:s3:::${bucket.getName()}`, - `arn:aws:s3:::${bucket.getName()}/*`, - ], + Action: ['s3:PutObjectRetention'], + Resource: [`arn:aws:s3:::${bucket.getName()}`, `arn:aws:s3:::${bucket.getName()}/*`], Condition: { NumericGreaterThan: { 's3:object-lock-remaining-retention-days': 10, @@ -626,10 +613,7 @@ describe('bucket policy authorization', () => { CanonicalUser: [altAcctCanonicalId], }, Action: 's3:*', - Resource: [ - `arn:aws:s3:::${bucket.getName()}`, - `arn:aws:s3:::${bucket.getName()}/*`, - ], + Resource: [`arn:aws:s3:::${bucket.getName()}`, `arn:aws:s3:::${bucket.getName()}/*`], Condition: { IpAddress: { 'aws:SourceIp': '123.123.123.123', @@ -692,11 +676,17 @@ describe('bucket policy authorization', () => { requestParams[t.requestConditionKey] = t.conditionValue; const request = new DummyRequest(requestParams); - const results = isObjAuthorized(bucket, object, t.requestType, - altAcctCanonicalId, altAcctAuthInfo, log, request); + const results = isObjAuthorized( + bucket, + object, + t.requestType, + altAcctCanonicalId, + altAcctAuthInfo, + log, + request, + ); assert.strictEqual(results, t.expectedVerdict); - } - ); + }); }); }); }); diff --git a/tests/unit/api/bucketPutACL.js b/tests/unit/api/bucketPutACL.js index 7c08b760ca..44953126c7 100644 --- a/tests/unit/api/bucketPutACL.js +++ b/tests/unit/api/bucketPutACL.js @@ -17,21 +17,15 @@ const testBucketPutRequest = { headers: { host: `${bucketName}.s3.amazonaws.com` }, url: '/', }; -const canonicalIDforSample1 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; -const canonicalIDforSample2 = - '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; +const canonicalIDforSample1 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const canonicalIDforSample2 = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2bf'; const invalidIds = { 'too short': 'id="invalid_id"', - 'too long': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2aaa', - 'only numbers': - 'id=0000000000000000000000000000000000000000000000000000000000000000', - 'only letters': - 'id=abcdefabcdefabcdefabcdefabcdefacbdefabcdefabcdefabcdefabcdefabcd', - 'non-hex letters': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ZZ', + 'too long': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2aaa', + 'only numbers': 'id=0000000000000000000000000000000000000000000000000000000000000000', + 'only letters': 'id=abcdefabcdefabcdefabcdefabcdefacbdefabcdefabcdefabcdefabcdefabcd', + 'non-hex letters': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2ZZ', }; describe('putBucketACL API', () => { @@ -65,7 +59,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'not-a-valid-option', }, url: '/?acl', @@ -84,7 +78,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read-write', }, url: '/?acl', @@ -100,13 +94,12 @@ describe('putBucketACL API', () => { }); }); - it('should set a canned public-read ACL followed by ' - + 'a canned authenticated-read ACL', done => { + it('should set a canned public-read ACL followed by ' + 'a canned authenticated-read ACL', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read', }, url: '/?acl', @@ -117,7 +110,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'authenticated-read', }, url: '/?acl', @@ -131,8 +124,7 @@ describe('putBucketACL API', () => { bucketPutACL(authInfo, testACLRequest2, log, err => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { - assert.strictEqual(md.getAcl().Canned, - 'authenticated-read'); + assert.strictEqual(md.getAcl().Canned, 'authenticated-read'); done(); }); }); @@ -140,13 +132,12 @@ describe('putBucketACL API', () => { }); }); - it('should set a canned private ACL ' + - 'followed by a log-delivery-write ACL', done => { + it('should set a canned private ACL ' + 'followed by a log-delivery-write ACL', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'private', }, url: '/?acl', @@ -157,7 +148,7 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'log-delivery-write', }, url: '/?acl', @@ -172,8 +163,7 @@ describe('putBucketACL API', () => { bucketPutACL(authInfo, testACLRequest2, log, err => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { - assert.strictEqual(md.getAcl().Canned, - 'log-delivery-write'); + assert.strictEqual(md.getAcl().Canned, 'log-delivery-write'); done(); }); }); @@ -186,18 +176,13 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: '/?acl', query: { acl: '' }, @@ -207,95 +192,77 @@ describe('putBucketACL API', () => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getAcl().WRITE[0], constants.publicId); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample2) > -1); - assert(md.getAcl().READ_ACP - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().WRITE_ACP - .indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().READ_ACP.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().WRITE_ACP.indexOf(canonicalIDforSample2) > -1); done(); }); }); }); - it('should set all ACLs sharing the same email in request headers', - done => { - const testACLRequest = { - bucketName, - namespace, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', - 'x-amz-grant-read': - 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-write': - 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', - }, - url: '/?acl', - query: { acl: '' }, - actionImplicitDenies: false, - }; - bucketPutACL(authInfo, testACLRequest, log, err => { - assert.strictEqual(err, undefined); - metadata.getBucket(bucketName, log, (err, md) => { - assert(md.getAcl().WRITE.indexOf(canonicalIDforSample1) - > -1); - assert(md.getAcl().READ.indexOf(canonicalIDforSample1) - > -1); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().FULL_CONTROL - .indexOf(canonicalIDforSample2) > -1); - assert(md.getAcl().READ_ACP - .indexOf(canonicalIDforSample1) > -1); - assert(md.getAcl().WRITE_ACP - .indexOf(canonicalIDforSample2) > -1); - done(); - }); + it('should set all ACLs sharing the same email in request headers', done => { + const testACLRequest = { + bucketName, + namespace, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', + 'x-amz-grant-read': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-write': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', + }, + url: '/?acl', + query: { acl: '' }, + actionImplicitDenies: false, + }; + bucketPutACL(authInfo, testACLRequest, log, err => { + assert.strictEqual(err, undefined); + metadata.getBucket(bucketName, log, (err, md) => { + assert(md.getAcl().WRITE.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().READ.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().FULL_CONTROL.indexOf(canonicalIDforSample2) > -1); + assert(md.getAcl().READ_ACP.indexOf(canonicalIDforSample1) > -1); + assert(md.getAcl().WRITE_ACP.indexOf(canonicalIDforSample2) > -1); + done(); }); }); + }); Object.keys(invalidIds).forEach(idType => { - it('should return an error if grantee canonical ID provided in ACL ' + - `request invalid because ${idType}`, done => { - const testACLRequest = { - bucketName, - namespace, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': invalidIds[idType], - }, - url: '/?acl', - query: { acl: '' }, - actionImplicitDenies: false, - }; - return bucketPutACL(authInfo, testACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); - }); + it( + 'should return an error if grantee canonical ID provided in ACL ' + `request invalid because ${idType}`, + done => { + const testACLRequest = { + bucketName, + namespace, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': invalidIds[idType], + }, + url: '/?acl', + query: { acl: '' }, + actionImplicitDenies: false, + }; + return bucketPutACL(authInfo, testACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); + }); + }, + ); }); - it('should return an error if invalid email ' + - 'provided in ACL header request', done => { + it('should return an error if invalid email ' + 'provided in ACL header request', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="nonexistentEmail@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="nonexistentEmail@sampling.com"', }, url: '/?acl', query: { acl: '' }, @@ -313,49 +280,50 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'FULL_CONTROL' + - '' + - '' + - '' + - `${constants.publicId}` + - '' + - 'READ' + - '' + - '' + - '' + - `${constants.logId}` + - '' + - 'WRITE' + - '' + - '' + - '' + - 'sampleaccount1@sampling.com' + - '' + - '' + - 'WRITE_ACP' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbacedfd' + - '6e09d98eacf8f8d5218e7cd47ef2bf' + - '' + - 'READ_ACP' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'FULL_CONTROL' + + '' + + '' + + '' + + `${constants.publicId}` + + '' + + 'READ' + + '' + + '' + + '' + + `${constants.logId}` + + '' + + 'WRITE' + + '' + + '' + + '' + + 'sampleaccount1@sampling.com' + + '' + + '' + + 'WRITE_ACP' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbacedfd' + + '6e09d98eacf8f8d5218e7cd47ef2bf' + + '' + + 'READ_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -366,14 +334,11 @@ describe('putBucketACL API', () => { assert.strictEqual(err, undefined); metadata.getBucket(bucketName, log, (err, md) => { assert.strictEqual(md.getAcl().Canned, ''); - assert.strictEqual(md.getAcl().FULL_CONTROL[0], - canonicalIDforSample1); + assert.strictEqual(md.getAcl().FULL_CONTROL[0], canonicalIDforSample1); assert.strictEqual(md.getAcl().READ[0], constants.publicId); assert.strictEqual(md.getAcl().WRITE[0], constants.logId); - assert.strictEqual(md.getAcl().WRITE_ACP[0], - canonicalIDforSample1); - assert.strictEqual(md.getAcl().READ_ACP[0], - canonicalIDforSample2); + assert.strictEqual(md.getAcl().WRITE_ACP[0], canonicalIDforSample1); + assert.strictEqual(md.getAcl().READ_ACP[0], canonicalIDforSample2); done(); }); }); @@ -384,14 +349,15 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -412,19 +378,19 @@ describe('putBucketACL API', () => { }); }); - it('should not be able to set ACLs without AccessControlList section', - done => { + it('should not be able to set ACLs without AccessControlList section', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -442,33 +408,34 @@ describe('putBucketACL API', () => { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'FULL_CONTROL' + - '' + - '' + - '' + - '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'READ' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'FULL_CONTROL' + + '' + + '' + + '' + + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'READ' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -481,27 +448,27 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid grantee user ID ' + - 'provided in ACL request body', done => { + it('should return an error if invalid grantee user ID ' + 'provided in ACL request body', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'invalid_id' + - '' + - 'READ_ACP' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'invalid_id' + + '' + + 'READ_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -514,27 +481,27 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid email ' + - 'address provided in ACLs set out in request body', done => { + it('should return an error if invalid email ' + 'address provided in ACLs set out in request body', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -546,31 +513,31 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if xml provided does not match s3 ' - + 'scheme for setting ACLs', done => { + it('should return an error if xml provided does not match s3 ' + 'scheme for setting ACLs', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, /** XML below uses the term "PowerGrant" instead of - * "Grant" which is part of the s3 xml scheme for ACLs - * so an error should be returned - */ - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + - '' + - '' + + * "Grant" which is part of the s3 xml scheme for ACLs + * so an error should be returned + */ + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -583,51 +550,54 @@ describe('putBucketACL API', () => { }); }); - - it('should return an error if xml provided does not match s3 ' - + 'scheme for setting ACLs using multiple Grant section', done => { - const testACLRequest = { - bucketName, - namespace, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - /** XML below uses the term "PowerGrant" instead of - * "Grant" which is part of the s3 xml scheme for ACLs - * so an error should be returned - */ - post: ' { + const testACLRequest = { + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + /** XML below uses the term "PowerGrant" instead of + * "Grant" which is part of the s3 xml scheme for ACLs + * so an error should be returned + */ + post: + '' + - '' + + '' + '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + 'OwnerDisplayName' + - '' + - '' + + '' + + '' + '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - 'FULL_CONTROL' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + 'FULL_CONTROL' + '' + '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + '' + - '' + - '', - url: '/?acl', - query: { acl: '' }, - actionImplicitDenies: false, - }; + '' + + '', + url: '/?acl', + query: { acl: '' }, + actionImplicitDenies: false, + }; - bucketPutACL(authInfo, testACLRequest, log, err => { - assert.strictEqual(err.is.MalformedACLError, true); - done(); - }); - }); + bucketPutACL(authInfo, testACLRequest, log, err => { + assert.strictEqual(err.is.MalformedACLError, true); + done(); + }); + }, + ); it('should return an error if malformed xml provided', done => { const testACLRequest = { @@ -639,20 +609,20 @@ describe('putBucketACL API', () => { post: { '' + - '' + + '' + '79a59df900b949e55d96a1e698fbaced' + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + 'OwnerDisplayName' + - '' + - '' + + '' + + '' + '' + - '' + - 'xyz@amazon.com' + - '' + - 'WRITE_ACP' + + '' + + 'xyz@amazon.com' + + '' + + 'WRITE_ACP' + '' + - '' + - '', + '' + + '', }, url: '/?acl', query: { acl: '' }, @@ -665,29 +635,29 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid group ' + - 'uri provided in ACLs set out in request body', done => { + it('should return an error if invalid group ' + 'uri provided in ACLs set out in request body', done => { const testACLRequest = { bucketName, namespace, headers: { host: `${bucketName}.s3.amazonaws.com` }, // URI in grant below is not valid group URI for s3 - post: '' + - '' + - '79a59df900b949e55d96a1e698fbaced' + - 'fd6e09d98eacf8f8d5218e7cd47ef2be' + - 'OwnerDisplayName' + - '' + - '' + - '' + - '' + - 'http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP' + - '' + - 'READ' + - '' + - '' + + post: + '' + + '' + + '79a59df900b949e55d96a1e698fbaced' + + 'fd6e09d98eacf8f8d5218e7cd47ef2be' + + 'OwnerDisplayName' + + '' + + '' + + '' + + '' + + 'http://acs.amazonaws.com/groups/' + + 'global/NOTAVALIDGROUP' + + '' + + 'READ' + + '' + + '' + '', url: '/?acl', query: { acl: '' }, @@ -700,16 +670,13 @@ describe('putBucketACL API', () => { }); }); - it('should return an error if invalid group uri' + - 'provided in ACL header request', done => { + it('should return an error if invalid group uri' + 'provided in ACL header request', done => { const testACLRequest = { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'x-amz-grant-full-control': - 'uri="http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP"', + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP"', }, url: '/?acl', query: { acl: '' }, diff --git a/tests/unit/api/bucketPutCors.js b/tests/unit/api/bucketPutCors.js index fc9e7bdcf2..e8c381d810 100644 --- a/tests/unit/api/bucketPutCors.js +++ b/tests/unit/api/bucketPutCors.js @@ -3,13 +3,8 @@ const { errors } = require('arsenal'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); -const { _validator, parseCorsXml } - = require('../../../lib/api/apiUtils/bucket/bucketCors'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - CorsConfigTester } - = require('../helpers'); +const { _validator, parseCorsXml } = require('../../../lib/api/apiUtils/bucket/bucketCors'); +const { cleanup, DummyRequestLogger, makeAuthInfo, CorsConfigTester } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -31,13 +26,14 @@ function _testPutBucketCors(authInfo, request, log, errCode, cb) { } function _generateSampleXml(value) { - const xml = '' + - '' + - 'PUT' + - 'www.example.com' + - `${value}` + - '' + - ''; + const xml = + '' + + '' + + 'PUT' + + 'www.example.com' + + `${value}` + + '' + + ''; return xml; } @@ -49,10 +45,9 @@ describe('putBucketCORS API', () => { }); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with cors resource', done => { + it("should update a bucket's metadata with cors resource", done => { const corsUtil = new CorsConfigTester(); - const testBucketPutCorsRequest = corsUtil - .createBucketCorsRequest('PUT', bucketName); + const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName); bucketPutCors(authInfo, testBucketPutCorsRequest, log, err => { if (err) { process.stdout.write(`Err putting bucket cors ${err}`); @@ -73,18 +68,14 @@ describe('putBucketCORS API', () => { it('should return MalformedXML if body greater than 64KB', done => { const corsUtil = new CorsConfigTester(); const body = Buffer.alloc(65537); // 64 * 1024 = 65536 bytes - const testBucketPutCorsRequest = corsUtil - .createBucketCorsRequest('PUT', bucketName, body); - _testPutBucketCors(authInfo, testBucketPutCorsRequest, - log, 'MalformedXML', done); + const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName, body); + _testPutBucketCors(authInfo, testBucketPutCorsRequest, log, 'MalformedXML', done); }); it('should return InvalidRequest if more than one MaxAgeSeconds', done => { const corsUtil = new CorsConfigTester({ maxAgeSeconds: [60, 6000] }); - const testBucketPutCorsRequest = corsUtil - .createBucketCorsRequest('PUT', bucketName); - _testPutBucketCors(authInfo, testBucketPutCorsRequest, - log, 'MalformedXML', done); + const testBucketPutCorsRequest = corsUtil.createBucketCorsRequest('PUT', bucketName); + _testPutBucketCors(authInfo, testBucketPutCorsRequest, log, 'MalformedXML', done); }); }); @@ -95,8 +86,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { const expectedResults = [true, true, false]; for (let i = 0; i < testStrings.length; i++) { - const result = _validator - .validateNumberWildcards(testStrings[i]); + const result = _validator.validateNumberWildcards(testStrings[i]); assert.strictEqual(result, expectedResults[i]); } done(); @@ -117,8 +107,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { it('should return MalformedXML if more than one ID per rule', done => { const testValue = 'testid'; - const xml = _generateSampleXml(`${testValue}` + - `${testValue}`); + const xml = _generateSampleXml(`${testValue}` + `${testValue}`); parseCorsXml(xml, log, err => { assert(err, 'Expected error but found none'); assert.strictEqual(err.is.MalformedXML, true); @@ -149,8 +138,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { describe('validateMaxAgeSeconds ', () => { it('should validate successfully for valid value', done => { const testValue = 60; - const xml = _generateSampleXml(`${testValue}` + - ''); + const xml = _generateSampleXml(`${testValue}` + ''); parseCorsXml(xml, log, (err, result) => { assert.strictEqual(err, null, `Found unexpected err ${err}`); assert.strictEqual(typeof result[0].maxAgeSeconds, 'number'); @@ -159,12 +147,11 @@ describe('PUT bucket cors :: helper validation functions ', () => { }); }); - it('should return MalformedXML if more than one MaxAgeSeconds ' + - 'per rule', done => { + it('should return MalformedXML if more than one MaxAgeSeconds ' + 'per rule', done => { const testValue = '60'; const xml = _generateSampleXml( - `${testValue}` + - `${testValue}`); + `${testValue}` + `${testValue}`, + ); parseCorsXml(xml, log, err => { assert(err, 'Expected error but found none'); assert.strictEqual(err.is.MalformedXML, true); @@ -174,8 +161,7 @@ describe('PUT bucket cors :: helper validation functions ', () => { it('should validate & return undefined if empty value', done => { const testValue = ''; - const xml = _generateSampleXml(`${testValue}` + - ''); + const xml = _generateSampleXml(`${testValue}` + ''); parseCorsXml(xml, log, (err, result) => { assert.strictEqual(err, null, `Found unexpected err ${err}`); assert.strictEqual(result[0].MaxAgeSeconds, undefined); diff --git a/tests/unit/api/bucketPutEncryption.js b/tests/unit/api/bucketPutEncryption.js index b635973345..8fbae168f1 100644 --- a/tests/unit/api/bucketPutEncryption.js +++ b/tests/unit/api/bucketPutEncryption.js @@ -9,7 +9,6 @@ const bucketPutEncryption = require('../../../lib/api/bucketPutEncryption'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const { templateSSEConfig, templateRequest, getSSEConfig } = require('../utils/bucketEncryption'); - const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); const bucketName = 'bucketname'; @@ -44,26 +43,36 @@ describe('bucketPutEncryption API', () => { }); it('should reject a config with no Rule', done => { - bucketPutEncryption(authInfo, templateRequest(bucketName, - { post: ` + bucketPutEncryption( + authInfo, + templateRequest(bucketName, { + post: ` `, - }), log, err => { - assert.strictEqual(err.is.MalformedXML, true); - done(); - }); + }), + log, + err => { + assert.strictEqual(err.is.MalformedXML, true); + done(); + }, + ); }); it('should reject a config with no ApplyServerSideEncryptionByDefault section', done => { - bucketPutEncryption(authInfo, templateRequest(bucketName, - { post: ` + bucketPutEncryption( + authInfo, + templateRequest(bucketName, { + post: ` `, - }), log, err => { - assert.strictEqual(err.is.MalformedXML, true); - done(); - }); + }), + log, + err => { + assert.strictEqual(err.is.MalformedXML, true); + done(); + }, + ); }); it('should reject a config with no SSEAlgorithm', done => { @@ -170,8 +179,9 @@ describe('bucketPutEncryption API', () => { }); }); - it('should update SSEAlgorithm if existing SSEAlgorithm is AES256, ' + - 'new SSEAlgorithm is aws:kms and no KMSMasterKeyID is provided', + it( + 'should update SSEAlgorithm if existing SSEAlgorithm is AES256, ' + + 'new SSEAlgorithm is aws:kms and no KMSMasterKeyID is provided', done => { const post = templateSSEConfig({ algorithm: 'AES256' }); bucketPutEncryption(authInfo, templateRequest(bucketName, { post }), log, err => { @@ -180,7 +190,10 @@ describe('bucketPutEncryption API', () => { assert.ifError(err); const { masterKeyId } = sseInfo; const newConf = templateSSEConfig({ algorithm: 'aws:kms' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, + return bucketPutEncryption( + authInfo, + templateRequest(bucketName, { post: newConf }), + log, err => { assert.ifError(err); return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { @@ -192,11 +205,12 @@ describe('bucketPutEncryption API', () => { }); done(); }); - } + }, ); }); }); - }); + }, + ); it('should update SSEAlgorithm to aws:kms and set KMSMasterKeyID', done => { const post = templateSSEConfig({ algorithm: 'AES256' }); @@ -379,15 +393,13 @@ describe('bucketPutEncryption API with account level encryption', () => { assert.ifError(err); assert.deepStrictEqual(sseInfo, expectedSseInfo); const newConf = templateSSEConfig({ algorithm: 'AES256' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, expectedSseInfo); - done(); - }); - } - ); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, expectedSseInfo); + done(); + }); + }); }); }); }); @@ -407,22 +419,20 @@ describe('bucketPutEncryption API with account level encryption', () => { }); const keyId = '12345'; const newConf = templateSSEConfig({ algorithm: 'aws:kms', keyId }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, { - cryptoScheme: 1, - algorithm: 'aws:kms', - mandatory: true, - masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, - configuredMasterKeyId: `${arnPrefix}${keyId}`, - isAccountEncryptionEnabled: true, - }); - done(); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, { + cryptoScheme: 1, + algorithm: 'aws:kms', + mandatory: true, + masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, + configuredMasterKeyId: `${arnPrefix}${keyId}`, + isAccountEncryptionEnabled: true, }); - } - ); + done(); + }); + }); }); }); }); @@ -441,21 +451,19 @@ describe('bucketPutEncryption API with account level encryption', () => { isAccountEncryptionEnabled: true, }); const newConf = templateSSEConfig({ algorithm: 'AES256' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, { - cryptoScheme: 1, - algorithm: 'AES256', - mandatory: true, - masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, - isAccountEncryptionEnabled: true, - }); - done(); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, { + cryptoScheme: 1, + algorithm: 'AES256', + mandatory: true, + masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, + isAccountEncryptionEnabled: true, }); - } - ); + done(); + }); + }); }); }); }); @@ -474,21 +482,19 @@ describe('bucketPutEncryption API with account level encryption', () => { configuredMasterKeyId: `${arnPrefix}${keyId}`, }); const newConf = templateSSEConfig({ algorithm: 'AES256' }); - return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, - err => { - assert.ifError(err); - return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { - assert.deepStrictEqual(updatedSSEInfo, { - cryptoScheme: 1, - algorithm: 'AES256', - mandatory: true, - masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, - isAccountEncryptionEnabled: true, - }); - done(); + return bucketPutEncryption(authInfo, templateRequest(bucketName, { post: newConf }), log, err => { + assert.ifError(err); + return getSSEConfig(bucketName, log, (err, updatedSSEInfo) => { + assert.deepStrictEqual(updatedSSEInfo, { + cryptoScheme: 1, + algorithm: 'AES256', + mandatory: true, + masterKeyId: `${arnPrefix}${accountLevelMasterKeyId}`, + isAccountEncryptionEnabled: true, }); - } - ); + done(); + }); + }); }); }); }); @@ -497,8 +503,9 @@ describe('bucketPutEncryption API with account level encryption', () => { describe('bucketPutEncryption API with failed vault service', () => { beforeEach(done => { sinon.stub(inMemory, 'supportsDefaultKeyPerAccount').value(true); - sinon.stub(vault, 'getOrCreateEncryptionKeyId').callsFake((accountCanonicalId, log, cb) => - cb(errors.ServiceFailure)); + sinon + .stub(vault, 'getOrCreateEncryptionKeyId') + .callsFake((accountCanonicalId, log, cb) => cb(errors.ServiceFailure)); bucketPut(authInfo, bucketPutRequest, log, done); }); diff --git a/tests/unit/api/bucketPutLifecycle.js b/tests/unit/api/bucketPutLifecycle.js index b3cd0071ec..2468cfe38b 100644 --- a/tests/unit/api/bucketPutLifecycle.js +++ b/tests/unit/api/bucketPutLifecycle.js @@ -2,12 +2,8 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutLifecycle = require('../../../lib/api/bucketPutLifecycle'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); -const { getLifecycleRequest, getLifecycleXml } = - require('../utils/lifecycleHelpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const { getLifecycleRequest, getLifecycleXml } = require('../utils/lifecycleHelpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -82,9 +78,8 @@ describe('putBucketLifecycle API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with lifecycle config obj', done => { - const testPutLifecycleRequest = getLifecycleRequest(bucketName, - getLifecycleXml()); + it("should update a bucket's metadata with lifecycle config obj", done => { + const testPutLifecycleRequest = getLifecycleRequest(bucketName, getLifecycleXml()); bucketPutLifecycle(authInfo, testPutLifecycleRequest, log, err => { if (err) { process.stdout.write(`Err putting lifecycle config ${err}`); @@ -95,10 +90,8 @@ describe('putBucketLifecycle API', () => { process.stdout.write(`Err retrieving bucket MD ${err}`); return done(err); } - const bucketLifecycleConfig = - bucket.getLifecycleConfiguration(); - assert.deepStrictEqual( - bucketLifecycleConfig, expectedLifecycleConfig); + const bucketLifecycleConfig = bucket.getLifecycleConfiguration(); + assert.deepStrictEqual(bucketLifecycleConfig, expectedLifecycleConfig); return done(); }); }); diff --git a/tests/unit/api/bucketPutLogging.js b/tests/unit/api/bucketPutLogging.js index 80b26c9a3d..8e81ccce5d 100644 --- a/tests/unit/api/bucketPutLogging.js +++ b/tests/unit/api/bucketPutLogging.js @@ -46,18 +46,22 @@ function createLoggingRequest(bucketName, post, headers = {}) { } function createValidLoggingXML(targetBucket, targetPrefix = 'logs/') { - return '' + + return ( + '' + '' + '' + `${targetBucket}` + `${targetPrefix}` + '' + - ''; + '' + ); } function createEmptyLoggingXML() { - return '' + - ''; + return ( + '' + + '' + ); } describe('bucketPutLogging API', () => { @@ -189,7 +193,8 @@ describe('bucketPutLogging API', () => { }); it('should return error for malformed XML - missing closing tag', done => { - const malformedXML = '' + + const malformedXML = + '' + '' + '' + `${targetBucket}` + @@ -206,7 +211,8 @@ describe('bucketPutLogging API', () => { }); it('should return error for malformed XML - invalid structure', done => { - const malformedXML = '' + + const malformedXML = + '' + '' + '' + 'invalid' + // Invalid tag @@ -234,7 +240,8 @@ describe('bucketPutLogging API', () => { }); it('should return NotImplemented error when TargetGrants is present', done => { - const loggingXMLWithGrants = '' + + const loggingXMLWithGrants = + '' + '' + '' + `${targetBucket}` + diff --git a/tests/unit/api/bucketPutNotification.js b/tests/unit/api/bucketPutNotification.js index 42456fba5b..cca311b78f 100644 --- a/tests/unit/api/bucketPutNotification.js +++ b/tests/unit/api/bucketPutNotification.js @@ -2,10 +2,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutNotification = require('../../../lib/api/bucketPutNotification'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -22,11 +19,7 @@ const expectedNotifConfig = { queueConfig: [ { id: 'notification-id', - events: [ - 's3:ObjectCreated:*', - 's3:ObjectTagging:*', - 's3:ObjectAcl:Put', - ], + events: ['s3:ObjectCreated:*', 's3:ObjectTagging:*', 's3:ObjectAcl:Put'], queueArn: 'arn:scality:bucketnotif:::target1', filterRules: undefined, }, @@ -34,16 +27,18 @@ const expectedNotifConfig = { }; function getNotifRequest(empty) { - const queueConfig = empty ? '' : - '' + - 'notification-id' + - 'arn:scality:bucketnotif:::target1' + - 's3:ObjectCreated:*' + - 's3:ObjectTagging:*' + - 's3:ObjectAcl:Put' + - ''; + const queueConfig = empty + ? '' + : '' + + 'notification-id' + + 'arn:scality:bucketnotif:::target1' + + 's3:ObjectCreated:*' + + 's3:ObjectTagging:*' + + 's3:ObjectAcl:Put' + + ''; - const notifXml = '' + + const notifXml = + '' + `${queueConfig}` + ''; diff --git a/tests/unit/api/bucketPutObjectLock.js b/tests/unit/api/bucketPutObjectLock.js index e048bb40f2..ca51f163fe 100644 --- a/tests/unit/api/bucketPutObjectLock.js +++ b/tests/unit/api/bucketPutObjectLock.js @@ -2,10 +2,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutObjectLock = require('../../../lib/api/bucketPutObjectLock'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, -} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -18,7 +15,8 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const objectLockXml = '' + 'Enabled' + '' + @@ -57,21 +55,20 @@ describe('putBucketObjectLock API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => bucketPut(authInfo, bucketObjLockRequest, log, done)); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with object lock config', done => { + it("should update a bucket's metadata with object lock config", done => { bucketPutObjectLock(authInfo, putObjLockRequest, log, err => { assert.ifError(err); return metadata.getBucket(bucketName, log, (err, bucket) => { assert.ifError(err); - const bucketObjectLockConfig = bucket. - getObjectLockConfiguration(); - assert.deepStrictEqual( - bucketObjectLockConfig, expectedObjectLockConfig); + const bucketObjectLockConfig = bucket.getObjectLockConfiguration(); + assert.deepStrictEqual(bucketObjectLockConfig, expectedObjectLockConfig); return done(); }); }); diff --git a/tests/unit/api/bucketPutPolicy.js b/tests/unit/api/bucketPutPolicy.js index b135ef09f7..c3ef92e2a9 100644 --- a/tests/unit/api/bucketPutPolicy.js +++ b/tests/unit/api/bucketPutPolicy.js @@ -2,10 +2,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -48,9 +45,8 @@ describe('putBucketPolicy API', () => { }); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with bucket policy obj', done => { - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), - log, err => { + it("should update a bucket's metadata with bucket policy obj", done => { + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { if (err) { process.stdout.write(`Err putting bucket policy ${err}`); return done(err); @@ -67,11 +63,9 @@ describe('putBucketPolicy API', () => { }); }); - it('should return error if policy resource does not include bucket name', - done => { + it('should return error if policy resource does not include bucket name', done => { expectedBucketPolicy.Statement[0].Resource = 'arn:aws::s3:::badname'; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), - log, err => { + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.strictEqual(err.is.MalformedPolicy, true); assert.strictEqual(err.description, 'Policy has invalid resource'); return done(); @@ -79,10 +73,8 @@ describe('putBucketPolicy API', () => { }); it('should not return error if policy contains conditions', done => { - expectedBucketPolicy.Statement[0].Condition = - { IpAddress: { 'aws:SourceIp': '123.123.123.123' } }; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, - err => { + expectedBucketPolicy.Statement[0].Condition = { IpAddress: { 'aws:SourceIp': '123.123.123.123' } }; + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.ifError(err); done(); }); @@ -90,18 +82,15 @@ describe('putBucketPolicy API', () => { it('should return error if policy contains service principal', done => { expectedBucketPolicy.Statement[0].Principal = { Service: ['test.com'] }; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, - err => { + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.strictEqual(err.is.NotImplemented, true); done(); }); }); it('should return error if policy contains federated principal', done => { - expectedBucketPolicy.Statement[0].Principal = - { Federated: 'www.test.com' }; - bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, - err => { + expectedBucketPolicy.Statement[0].Principal = { Federated: 'www.test.com' }; + bucketPutPolicy(authInfo, getPolicyRequest(expectedBucketPolicy), log, err => { assert.strictEqual(err.is.NotImplemented, true); done(); }); diff --git a/tests/unit/api/bucketPutReplication.js b/tests/unit/api/bucketPutReplication.js index 4a19f5e24b..90b9fc4b84 100644 --- a/tests/unit/api/bucketPutReplication.js +++ b/tests/unit/api/bucketPutReplication.js @@ -1,14 +1,10 @@ const assert = require('assert'); -const bucketPutReplication = - require('../../../lib/api/bucketPutReplication'); +const bucketPutReplication = require('../../../lib/api/bucketPutReplication'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); -const { getReplicationConfiguration } = - require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); -const validateReplicationConfig = - require('../../../lib/api/apiUtils/bucket/validateReplicationConfig'); -const replicationUtils = - require('../../functional/aws-node-sdk/lib/utility/replication'); +const { getReplicationConfiguration } = require('../../../lib/api/apiUtils/bucket/getReplicationConfiguration'); +const validateReplicationConfig = require('../../../lib/api/apiUtils/bucket/validateReplicationConfig'); +const replicationUtils = require('../../functional/aws-node-sdk/lib/utility/replication'); const log = new DummyRequestLogger(); // Check for the expected error response code and status code. @@ -17,8 +13,10 @@ function checkError(xml, expectedErr, cb) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert(err.is[expectedErr], 'incorrect error response: should be ' + - `'Error: ${expectedErr}' but got '${err}'`); + assert( + err.is[expectedErr], + 'incorrect error response: should be ' + `'Error: ${expectedErr}' but got '${err}'`, + ); } return cb(); }); @@ -31,45 +29,42 @@ function checkGeneratedID(xml, cb) { return cb(err); } const id = res.rules[0].id; - assert.strictEqual(typeof id, 'string', 'expected rule ID to be ' + - `string but got ${typeof id}`); - assert.strictEqual(id.length, 48, 'expected rule ID to be a length ' + - `of 48 but got ${id.length}`); + assert.strictEqual(typeof id, 'string', 'expected rule ID to be ' + `string but got ${typeof id}`); + assert.strictEqual(id.length, 48, 'expected rule ID to be a length ' + `of 48 but got ${id.length}`); return cb(); }); } // Create replication configuration XML with an tag optionally omitted. function createReplicationXML(missingTag, tagValue) { - let Role = missingTag === 'Role' ? '' : - '' + - 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource' + - ''; + let Role = + missingTag === 'Role' + ? '' + : '' + + 'arn:aws:iam::account-id:role/src-resource,' + + 'arn:aws:iam::account-id:role/dest-resource' + + ''; Role = tagValue && tagValue.Role ? `${tagValue.Role}` : Role; let ID = missingTag === 'ID' ? '' : 'foo'; ID = tagValue && tagValue.ID === '' ? '' : ID; const Prefix = missingTag === 'Prefix' ? '' : 'foo'; const Status = missingTag === 'Status' ? '' : 'Enabled'; - const Bucket = missingTag === 'Bucket' ? '' : - 'arn:aws:s3:::destination-bucket'; - let StorageClass = missingTag === 'StorageClass' ? '' : - 'STANDARD'; - StorageClass = tagValue && tagValue.StorageClass ? - `${tagValue.StorageClass}` : StorageClass; - const Destination = missingTag === 'Destination' ? '' : - `${Bucket + StorageClass}`; - const Rule = missingTag === 'Rule' ? '' : - `${ID + Prefix + Status + Destination}`; + const Bucket = missingTag === 'Bucket' ? '' : 'arn:aws:s3:::destination-bucket'; + let StorageClass = missingTag === 'StorageClass' ? '' : 'STANDARD'; + StorageClass = + tagValue && tagValue.StorageClass ? `${tagValue.StorageClass}` : StorageClass; + const Destination = missingTag === 'Destination' ? '' : `${Bucket + StorageClass}`; + const Rule = missingTag === 'Rule' ? '' : `${ID + Prefix + Status + Destination}`; const content = missingTag === null ? '' : `${Role}${Rule}`; - return '${content}` + - ''; + return ( + '${content}` + + '' + ); } -describe('\'getReplicationConfiguration\' function', () => { - it('should not return error when putting valid XML', done => - checkError(createReplicationXML(), null, done)); +describe("'getReplicationConfiguration' function", () => { + it('should not return error when putting valid XML', done => checkError(createReplicationXML(), null, done)); it('should not accept empty replication configuration', done => checkError(createReplicationXML(null), 'MalformedXML', done)); @@ -79,13 +74,13 @@ describe('\'getReplicationConfiguration\' function', () => { const xmlTag = prop === 'Rules' ? 'Rule' : prop; const xml = createReplicationXML(xmlTag); - it(`should not accept replication configuration without '${prop}'`, - done => checkError(xml, 'MalformedXML', done)); + it(`should not accept replication configuration without '${prop}'`, done => + checkError(xml, 'MalformedXML', done)); }); replicationUtils.optionalConfigProperties.forEach(prop => { - it(`should accept replication configuration without '${prop}'`, - done => checkError(createReplicationXML(prop), null, done)); + it(`should accept replication configuration without '${prop}'`, done => + checkError(createReplicationXML(prop), null, done)); }); it(`should accept replication configuration without 'Bucket' when there @@ -97,19 +92,18 @@ describe('\'getReplicationConfiguration\' function', () => { checkError(xml, null, done); }); - it("should create a rule 'ID' if omitted from the replication " + - 'configuration', done => { + it("should create a rule 'ID' if omitted from the replication " + 'configuration', done => { const xml = createReplicationXML('ID'); return checkGeneratedID(xml, done); }); - it('should create an \'ID\' if rule ID is \'\'', done => { + it("should create an 'ID' if rule ID is ''", done => { const xml = createReplicationXML(undefined, { ID: '' }); return checkGeneratedID(xml, done); }); }); -describe('\'validateReplicationConfig\' function', () => { +describe("'validateReplicationConfig' function", () => { const nonTransientBucket = { getLocationConstraint: () => 'us-east-1', }; @@ -117,59 +111,67 @@ describe('\'validateReplicationConfig\' function', () => { getLocationConstraint: () => 'transientfile', }; - it('should validate configuration when bucket location is ' + - 'not transient and preferred read location is not specified', () => { - const withoutPreferredRead = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: 'test-prefix', - enabled: true, - id: 'test-id', - storageClass: 'STANDARD,us-east-2', - }], - }; - const result = validateReplicationConfig(withoutPreferredRead, - nonTransientBucket); - assert.strictEqual(result, true); - }); - - it('should validate configuration when bucket location is transient ' + - 'and preferred read location is specified', () => { - const withPreferredRead = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: 'test-prefix', - enabled: true, - id: 'test-id', - storageClass: 'STANDARD,us-east-2:preferred_read', - }], - }; - const result = validateReplicationConfig(withPreferredRead, - transientBucket); - assert.strictEqual(result, true); - }); - - it('should not validate configuration when bucket location is ' + - 'transient and preferred read location is not specified', () => { - const withoutPreferredRead = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: 'test-prefix', - enabled: true, - id: 'test-id', - storageClass: 'STANDARD,us-east-2', - }], - }; - const result = validateReplicationConfig(withoutPreferredRead, - transientBucket); - assert.strictEqual(result, false); - }); + it( + 'should validate configuration when bucket location is ' + + 'not transient and preferred read location is not specified', + () => { + const withoutPreferredRead = { + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: 'test-prefix', + enabled: true, + id: 'test-id', + storageClass: 'STANDARD,us-east-2', + }, + ], + }; + const result = validateReplicationConfig(withoutPreferredRead, nonTransientBucket); + assert.strictEqual(result, true); + }, + ); + + it( + 'should validate configuration when bucket location is transient ' + 'and preferred read location is specified', + () => { + const withPreferredRead = { + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: 'test-prefix', + enabled: true, + id: 'test-id', + storageClass: 'STANDARD,us-east-2:preferred_read', + }, + ], + }; + const result = validateReplicationConfig(withPreferredRead, transientBucket); + assert.strictEqual(result, true); + }, + ); + + it( + 'should not validate configuration when bucket location is ' + + 'transient and preferred read location is not specified', + () => { + const withoutPreferredRead = { + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: 'test-prefix', + enabled: true, + id: 'test-id', + storageClass: 'STANDARD,us-east-2', + }, + ], + }; + const result = validateReplicationConfig(withoutPreferredRead, transientBucket); + assert.strictEqual(result, false); + }, + ); }); describe('bucketPutReplication API', () => { diff --git a/tests/unit/api/bucketPutTagging.js b/tests/unit/api/bucketPutTagging.js index 1d95729c46..4b6324b078 100644 --- a/tests/unit/api/bucketPutTagging.js +++ b/tests/unit/api/bucketPutTagging.js @@ -1,11 +1,7 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester, -} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const bucketPutTagging = require('../../../lib/api/bucketPutTagging'); const log = new DummyRequestLogger(); const authInfo = makeAuthInfo('accessKey1'); @@ -28,8 +24,7 @@ describe('putBucketTagging API', () => { it('should set tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { if (err) { process.stdout.write(`Err putting object tagging ${err}`); @@ -42,8 +37,7 @@ describe('putBucketTagging API', () => { it('should return access denied if the authorization check fails', done => { const taggingUtil = new TaggingConfigTester(); - const testBucketPutTaggingRequest = taggingUtil - .createBucketTaggingRequest('PUT', bucketName); + const testBucketPutTaggingRequest = taggingUtil.createBucketTaggingRequest('PUT', bucketName); const authInfo = makeAuthInfo('accessKey2'); bucketPutTagging(authInfo, testBucketPutTaggingRequest, log, err => { assert(err.AccessDenied); diff --git a/tests/unit/api/bucketPutVersioning.js b/tests/unit/api/bucketPutVersioning.js index 462fadcf74..7cb4ef24f0 100644 --- a/tests/unit/api/bucketPutVersioning.js +++ b/tests/unit/api/bucketPutVersioning.js @@ -6,46 +6,44 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutVersioning = require('../../../lib/api/bucketPutVersioning'); const bucketPutReplication = require('../../../lib/api/bucketPutReplication'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo } = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const xmlEnableVersioning = -'' + -'Enabled' + -''; + '' + + 'Enabled' + + ''; const xmlSuspendVersioning = -'' + -'Suspended' + -''; + '' + + 'Suspended' + + ''; const locConstraintVersioned = -'' + -'withversioning' + -''; + '' + + 'withversioning' + + ''; const locConstraintNonVersioned = -'' + -'withoutversioning' + -''; + '' + + 'withoutversioning' + + ''; const xmlReplicationConfiguration = -'' + + '' + 'arn:aws:iam::account-id:role/src-resource' + '' + - '' + - 'Enabled' + - '' + - 'arn:aws:s3:::destination-bucket' + - 'us-east-2' + - '' + + '' + + 'Enabled' + + '' + + 'arn:aws:s3:::destination-bucket' + + 'us-east-2' + + '' + '' + -''; + ''; -const externalVersioningErrorMessage = 'We do not currently support putting ' + -'a versioned object to a location-constraint of type Azure or GCP.'; +const externalVersioningErrorMessage = + 'We do not currently support putting ' + 'a versioned object to a location-constraint of type Azure or GCP.'; const log = new DummyRequestLogger(); const bucketName = 'bucketname'; @@ -94,55 +92,58 @@ describe('bucketPutVersioning API', () => { const tests = [ { - msg: 'should successfully enable versioning on location ' + - 'constraint with supportsVersioning set to true', + msg: + 'should successfully enable versioning on location ' + + 'constraint with supportsVersioning set to true', input: xmlEnableVersioning, output: { Status: 'Enabled' }, }, { - msg: 'should successfully suspend versioning on location ' + - 'constraint with supportsVersioning set to true', + msg: + 'should successfully suspend versioning on location ' + + 'constraint with supportsVersioning set to true', input: xmlSuspendVersioning, output: { Status: 'Suspended' }, }, ]; - tests.forEach(test => it(test.msg, done => { - const request = _putVersioningRequest(test.input); - bucketPutVersioning(authInfo, request, log, err => { - assert.ifError(err, - `Expected success, but got err: ${err}`); - metadata.getBucket(bucketName, log, (err, bucket) => { - assert.ifError(err, - `Expected success, but got err: ${err}`); - assert.deepStrictEqual(bucket._versioningConfiguration, - test.output); - done(); + tests.forEach(test => + it(test.msg, done => { + const request = _putVersioningRequest(test.input); + bucketPutVersioning(authInfo, request, log, err => { + assert.ifError(err, `Expected success, but got err: ${err}`); + metadata.getBucket(bucketName, log, (err, bucket) => { + assert.ifError(err, `Expected success, but got err: ${err}`); + assert.deepStrictEqual(bucket._versioningConfiguration, test.output); + done(); + }); }); - }); - })); + }), + ); it('should not suspend versioning on bucket with replication', done => { - async.series([ - // Enable versioning to allow putting a replication config. - next => { - const request = _putVersioningRequest(xmlEnableVersioning); - bucketPutVersioning(authInfo, request, log, next); - }, - // Put the replication config on the bucket. - next => { - const request = - _putReplicationRequest(xmlReplicationConfiguration); - bucketPutReplication(authInfo, request, log, next); - }, - // Attempt to suspend versioning. - next => { - const request = _putVersioningRequest(xmlSuspendVersioning); - bucketPutVersioning(authInfo, request, log, err => { - assert(err.is.InvalidBucketState); - next(); - }); - }, - ], done); + async.series( + [ + // Enable versioning to allow putting a replication config. + next => { + const request = _putVersioningRequest(xmlEnableVersioning); + bucketPutVersioning(authInfo, request, log, next); + }, + // Put the replication config on the bucket. + next => { + const request = _putReplicationRequest(xmlReplicationConfiguration); + bucketPutReplication(authInfo, request, log, next); + }, + // Attempt to suspend versioning. + next => { + const request = _putVersioningRequest(xmlSuspendVersioning); + bucketPutVersioning(authInfo, request, log, err => { + assert(err.is.InvalidBucketState); + next(); + }); + }, + ], + done, + ); }); }); @@ -154,28 +155,28 @@ describe('bucketPutVersioning API', () => { const tests = [ { - msg: 'should return error if enabling versioning on location ' + - 'constraint with supportsVersioning set to false', + msg: + 'should return error if enabling versioning on location ' + + 'constraint with supportsVersioning set to false', input: xmlEnableVersioning, - output: { error: errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage) }, + output: { error: errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage) }, }, { - msg: 'should return error if suspending versioning on ' + - ' location constraint with supportsVersioning set to false', + msg: + 'should return error if suspending versioning on ' + + ' location constraint with supportsVersioning set to false', input: xmlSuspendVersioning, - output: { error: errorInstances.NotImplemented.customizeDescription( - externalVersioningErrorMessage) }, + output: { error: errorInstances.NotImplemented.customizeDescription(externalVersioningErrorMessage) }, }, ]; - tests.forEach(test => it(test.msg, done => { - const putBucketVersioningRequest = - _putVersioningRequest(test.input); - bucketPutVersioning(authInfo, putBucketVersioningRequest, log, - err => { - assert.deepStrictEqual(err, test.output.error); - done(); - }); - })); + tests.forEach(test => + it(test.msg, done => { + const putBucketVersioningRequest = _putVersioningRequest(test.input); + bucketPutVersioning(authInfo, putBucketVersioningRequest, log, err => { + assert.deepStrictEqual(err, test.output.error); + done(); + }); + }), + ); }); }); diff --git a/tests/unit/api/bucketPutWebsite.js b/tests/unit/api/bucketPutWebsite.js index 45944910af..c2f46ee62f 100644 --- a/tests/unit/api/bucketPutWebsite.js +++ b/tests/unit/api/bucketPutWebsite.js @@ -3,13 +3,8 @@ const { parseString } = require('xml2js'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); -const { xmlContainsElem } - = require('../../../lib/api/apiUtils/bucket/bucketWebsite'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - WebsiteConfig } - = require('../helpers'); +const { xmlContainsElem } = require('../../../lib/api/apiUtils/bucket/bucketWebsite'); +const { cleanup, DummyRequestLogger, makeAuthInfo, WebsiteConfig } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); const log = new DummyRequestLogger(); @@ -41,12 +36,10 @@ describe('putBucketWebsite API', () => { beforeEach(done => bucketPut(authInfo, testBucketPutRequest, log, done)); afterEach(() => cleanup()); - it('should update a bucket\'s metadata with website config obj', done => { + it("should update a bucket's metadata with website config obj", done => { const config = new WebsiteConfig('index.html', 'error.html'); - config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, - { KeyPrefixEquals: 'docs/' }); - const testBucketPutWebsiteRequest = - _getPutWebsiteRequest(config.getXml()); + config.addRoutingRule({ ReplaceKeyPrefixWith: 'documents/' }, { KeyPrefixEquals: 'docs/' }); + const testBucketPutWebsiteRequest = _getPutWebsiteRequest(config.getXml()); bucketPutWebsite(authInfo, testBucketPutWebsiteRequest, log, err => { if (err) { process.stdout.write(`Err putting website config ${err}`); @@ -58,122 +51,122 @@ describe('putBucketWebsite API', () => { return done(err); } const bucketWebsiteConfig = bucket.getWebsiteConfiguration(); - assert.strictEqual(bucketWebsiteConfig._indexDocument, - config.IndexDocument.Suffix); - assert.strictEqual(bucketWebsiteConfig._errorDocument, - config.ErrorDocument.Key); - assert.strictEqual(bucketWebsiteConfig._routingRules[0] - ._condition.keyPrefixEquals, - config.RoutingRules[0].Condition.KeyPrefixEquals); - assert.strictEqual(bucketWebsiteConfig._routingRules[0] - ._redirect.replaceKeyPrefixWith, - config.RoutingRules[0].Redirect.ReplaceKeyPrefixWith); + assert.strictEqual(bucketWebsiteConfig._indexDocument, config.IndexDocument.Suffix); + assert.strictEqual(bucketWebsiteConfig._errorDocument, config.ErrorDocument.Key); + assert.strictEqual( + bucketWebsiteConfig._routingRules[0]._condition.keyPrefixEquals, + config.RoutingRules[0].Condition.KeyPrefixEquals, + ); + assert.strictEqual( + bucketWebsiteConfig._routingRules[0]._redirect.replaceKeyPrefixWith, + config.RoutingRules[0].Redirect.ReplaceKeyPrefixWith, + ); return done(); }); }); }); describe('helper functions', () => { - it('xmlContainsElem should return true if xml contains ' + - 'specified element', done => { - const xml = '' + - 'value' + - ''; + it('xmlContainsElem should return true if xml contains ' + 'specified element', done => { + const xml = '' + 'value' + ''; parseString(xml, (err, result) => { if (err) { process.stdout.write(`Unexpected err ${err} parsing xml`); return done(err); } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - 'Element'); + const containsRes = xmlContainsElem(result.Toplevel.Parent, 'Element'); assert.strictEqual(containsRes, true); return done(); }); }); - it('xmlContainsElem should return false if xml does not contain ' + - 'specified element', done => { - const xml = '' + - 'value' + - ''; + it('xmlContainsElem should return false if xml does not contain ' + 'specified element', done => { + const xml = '' + 'value' + ''; parseString(xml, (err, result) => { if (err) { process.stdout.write(`Unexpected err ${err} parsing xml`); return done(err); } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - 'Element'); + const containsRes = xmlContainsElem(result.Toplevel.Parent, 'Element'); assert.strictEqual(containsRes, false); return done(); }); }); - it('xmlContainsElem should return true if parent contains list of ' + - 'elements and isList is specified in options', done => { - const xml = '' + - 'value' + - 'value' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - 'Element', { isList: true }); - assert.strictEqual(containsRes, true); - return done(); - }); - }); - it('xmlContainsElem should return true if parent contains at least ' + - 'one of the elements specified, if multiple', done => { - const xml = '' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - ['ElementA', 'ElementB']); - assert.strictEqual(containsRes, true); - return done(); - }); - }); - it('xmlContainsElem should return false if parent contains only one ' + - 'of multiple elements specified and checkForAll specified in options', - done => { - const xml = '' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - ['ElementA', 'ElementB'], { checkForAll: true }); - assert.strictEqual(containsRes, false); - return done(); - }); - }); - it('xmlContainsElem should return true if parent contains all ' + - 'of multiple elements specified and checkForAll specified in options', - done => { - const xml = '' + - 'value' + - 'value' + - ''; - parseString(xml, (err, result) => { - if (err) { - process.stdout.write(`Unexpected err ${err} parsing xml`); - return done(err); - } - const containsRes = xmlContainsElem(result.Toplevel.Parent, - ['ElementA', 'ElementB'], { checkForAll: true }); - assert.strictEqual(containsRes, true); - return done(); - }); - }); + it( + 'xmlContainsElem should return true if parent contains list of ' + + 'elements and isList is specified in options', + done => { + const xml = + '' + + 'value' + + 'value' + + 'value' + + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, 'Element', { isList: true }); + assert.strictEqual(containsRes, true); + return done(); + }); + }, + ); + it( + 'xmlContainsElem should return true if parent contains at least ' + + 'one of the elements specified, if multiple', + done => { + const xml = '' + 'value' + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, ['ElementA', 'ElementB']); + assert.strictEqual(containsRes, true); + return done(); + }); + }, + ); + it( + 'xmlContainsElem should return false if parent contains only one ' + + 'of multiple elements specified and checkForAll specified in options', + done => { + const xml = '' + 'value' + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, ['ElementA', 'ElementB'], { + checkForAll: true, + }); + assert.strictEqual(containsRes, false); + return done(); + }); + }, + ); + it( + 'xmlContainsElem should return true if parent contains all ' + + 'of multiple elements specified and checkForAll specified in options', + done => { + const xml = + '' + + 'value' + + 'value' + + ''; + parseString(xml, (err, result) => { + if (err) { + process.stdout.write(`Unexpected err ${err} parsing xml`); + return done(err); + } + const containsRes = xmlContainsElem(result.Toplevel.Parent, ['ElementA', 'ElementB'], { + checkForAll: true, + }); + assert.strictEqual(containsRes, true); + return done(); + }); + }, + ); }); }); diff --git a/tests/unit/api/corsErrorHeaders.js b/tests/unit/api/corsErrorHeaders.js index 89216f4cdb..102a9f58eb 100644 --- a/tests/unit/api/corsErrorHeaders.js +++ b/tests/unit/api/corsErrorHeaders.js @@ -10,55 +10,42 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutCors = require('../../../lib/api/bucketPutCors'); const metadata = require('../../../lib/metadata/wrapper'); const DummyRequest = require('../DummyRequest'); -const { - CorsConfigTester, - DummyRequestLogger, - cleanup, - makeAuthInfo, -} = require('../helpers'); +const { CorsConfigTester, DummyRequestLogger, cleanup, makeAuthInfo } = require('../helpers'); const endpoints = [ { apiMethod: 'bucketGet', httpMethod: 'GET', url: '/', query: {} }, { apiMethod: 'bucketHead', httpMethod: 'HEAD', url: '/', query: {} }, { apiMethod: 'bucketDelete', httpMethod: 'DELETE', url: '/', query: {} }, - { apiMethod: 'bucketGetACL', httpMethod: 'GET', url: '/?acl', - query: { acl: '' } }, - { apiMethod: 'bucketGetCors', httpMethod: 'GET', url: '/?cors', - query: { cors: '' } }, - { apiMethod: 'bucketGetLifecycle', httpMethod: 'GET', url: '/?lifecycle', - query: { lifecycle: '' } }, - { apiMethod: 'bucketGetReplication', httpMethod: 'GET', - url: '/?replication', query: { replication: '' } }, - { apiMethod: 'bucketGetPolicy', httpMethod: 'GET', url: '/?policy', - query: { policy: '' } }, - { apiMethod: 'bucketGetVersioning', httpMethod: 'GET', url: '/?versioning', - query: { versioning: '' } }, - { apiMethod: 'bucketGetWebsite', httpMethod: 'GET', url: '/?website', - query: { website: '' } }, - { apiMethod: 'bucketGetTagging', httpMethod: 'GET', url: '/?tagging', - query: { tagging: '' } }, - { apiMethod: 'bucketGetEncryption', httpMethod: 'GET', url: '/?encryption', - query: { encryption: '' } }, - { apiMethod: 'bucketGetNotification', httpMethod: 'GET', - url: '/?notification', query: { notification: '' } }, - { apiMethod: 'bucketGetObjectLock', httpMethod: 'GET', - url: '/?object-lock', query: { 'object-lock': '' } }, - { apiMethod: 'bucketGetLocation', httpMethod: 'GET', url: '/?location', - query: { location: '' } }, - { apiMethod: 'objectGet', httpMethod: 'GET', url: '/obj', query: {}, - objectKey: 'obj' }, - { apiMethod: 'objectHead', httpMethod: 'HEAD', url: '/obj', query: {}, - objectKey: 'obj' }, - { apiMethod: 'objectDelete', httpMethod: 'DELETE', url: '/obj', query: {}, - objectKey: 'obj' }, - { apiMethod: 'objectGetLegalHold', httpMethod: 'GET', - url: '/obj?legal-hold', query: { 'legal-hold': '' }, - objectKey: 'obj' }, - { apiMethod: 'objectGetAttributes', httpMethod: 'GET', - url: '/obj?attributes', query: { attributes: '' }, - objectKey: 'obj' }, - { apiMethod: 'listMultipartUploads', httpMethod: 'GET', url: '/?uploads', - query: { uploads: '' } }, + { apiMethod: 'bucketGetACL', httpMethod: 'GET', url: '/?acl', query: { acl: '' } }, + { apiMethod: 'bucketGetCors', httpMethod: 'GET', url: '/?cors', query: { cors: '' } }, + { apiMethod: 'bucketGetLifecycle', httpMethod: 'GET', url: '/?lifecycle', query: { lifecycle: '' } }, + { apiMethod: 'bucketGetReplication', httpMethod: 'GET', url: '/?replication', query: { replication: '' } }, + { apiMethod: 'bucketGetPolicy', httpMethod: 'GET', url: '/?policy', query: { policy: '' } }, + { apiMethod: 'bucketGetVersioning', httpMethod: 'GET', url: '/?versioning', query: { versioning: '' } }, + { apiMethod: 'bucketGetWebsite', httpMethod: 'GET', url: '/?website', query: { website: '' } }, + { apiMethod: 'bucketGetTagging', httpMethod: 'GET', url: '/?tagging', query: { tagging: '' } }, + { apiMethod: 'bucketGetEncryption', httpMethod: 'GET', url: '/?encryption', query: { encryption: '' } }, + { apiMethod: 'bucketGetNotification', httpMethod: 'GET', url: '/?notification', query: { notification: '' } }, + { apiMethod: 'bucketGetObjectLock', httpMethod: 'GET', url: '/?object-lock', query: { 'object-lock': '' } }, + { apiMethod: 'bucketGetLocation', httpMethod: 'GET', url: '/?location', query: { location: '' } }, + { apiMethod: 'objectGet', httpMethod: 'GET', url: '/obj', query: {}, objectKey: 'obj' }, + { apiMethod: 'objectHead', httpMethod: 'HEAD', url: '/obj', query: {}, objectKey: 'obj' }, + { apiMethod: 'objectDelete', httpMethod: 'DELETE', url: '/obj', query: {}, objectKey: 'obj' }, + { + apiMethod: 'objectGetLegalHold', + httpMethod: 'GET', + url: '/obj?legal-hold', + query: { 'legal-hold': '' }, + objectKey: 'obj', + }, + { + apiMethod: 'objectGetAttributes', + httpMethod: 'GET', + url: '/obj?attributes', + query: { attributes: '' }, + objectKey: 'obj', + }, + { apiMethod: 'listMultipartUploads', httpMethod: 'GET', url: '/?uploads', query: { uploads: '' } }, ]; const bucketName = 'corserrorheaderstest'; @@ -95,24 +82,29 @@ function buildRequest(spec) { // DummyRequest is an http.IncomingMessage stream that emits 'end' // synchronously. We need that because callApiMethod's waterfall // waits for the request body on non-objectPut paths. - return new DummyRequest({ - bucketName, - objectKey: spec.objectKey, - headers: { - host: `${bucketName}.s3.amazonaws.com`, - origin, + return new DummyRequest( + { + bucketName, + objectKey: spec.objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + origin, + }, + url: spec.url, + query: spec.query, + method: spec.httpMethod, }, - url: spec.url, - query: spec.query, - method: spec.httpMethod, - }, Buffer.alloc(0)); + Buffer.alloc(0), + ); } function buildResponseSpy(sandbox) { const headers = {}; return { headers, - setHeader: sandbox.spy((k, v) => { headers[k.toLowerCase()] = v; }), + setHeader: sandbox.spy((k, v) => { + headers[k.toLowerCase()] = v; + }), getHeader: k => headers[k.toLowerCase()], }; } @@ -126,7 +118,9 @@ function buildLog(sandbox) { warn: sandbox.stub(), error: sandbox.stub(), fatal: sandbox.stub(), - end() { return this; }, + end() { + return this; + }, }; } @@ -146,75 +140,65 @@ describe('CORS headers on 403 auth failures (api.callApiMethod)', () => { afterEach(() => sandbox.restore()); endpoints.forEach(spec => { - it(`attaches CORS headers to 403 response for ${spec.apiMethod}`, - done => { - const request = buildRequest(spec); - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); - - api.callApiMethod(spec.apiMethod, request, response, log, - err => { - assert(err, 'expected an error'); - assert(err.is && err.is.AccessDenied, - `expected AccessDenied, got ${err.code}`); - // Either the callback surfaces CORS headers in one of - // its trailing args OR they have been set directly on - // the HTTP response. We assert on the response since - // that is what the HTTP transport ultimately sends. - const allowOrigin = response.getHeader( - 'access-control-allow-origin'); - assert(allowOrigin, - 'access-control-allow-origin missing from 403 ' - + `response for ${spec.apiMethod}`); - assert(response.getHeader( - 'access-control-allow-methods'), - 'access-control-allow-methods missing'); - done(); - }); - }); - }); - - it('does not attach CORS headers when Origin header is absent', - done => { - const request = buildRequest({ - apiMethod: 'bucketGet', httpMethod: 'GET', - url: '/', query: {}, - }); - delete request.headers.origin; + it(`attaches CORS headers to 403 response for ${spec.apiMethod}`, done => { + const request = buildRequest(spec); const response = buildResponseSpy(sandbox); const log = buildLog(sandbox); - api.callApiMethod('bucketGet', request, response, log, err => { - assert(err && err.is.AccessDenied); - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined); + api.callApiMethod(spec.apiMethod, request, response, log, err => { + assert(err, 'expected an error'); + assert(err.is && err.is.AccessDenied, `expected AccessDenied, got ${err.code}`); + // Either the callback surfaces CORS headers in one of + // its trailing args OR they have been set directly on + // the HTTP response. We assert on the response since + // that is what the HTTP transport ultimately sends. + const allowOrigin = response.getHeader('access-control-allow-origin'); + assert(allowOrigin, 'access-control-allow-origin missing from 403 ' + `response for ${spec.apiMethod}`); + assert(response.getHeader('access-control-allow-methods'), 'access-control-allow-methods missing'); done(); }); }); + }); - it('does not attach CORS headers when origin does not match any rule', - done => { - const request = buildRequest({ - apiMethod: 'bucketGet', httpMethod: 'GET', - url: '/', query: {}, - }); - request.headers.origin = 'http://not-allowed.test'; - // The bucket's CORS config allows any method from foo.test - // plus GET from *. Use PUT from a different origin so neither - // condition matches. - request.method = 'PUT'; - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); + it('does not attach CORS headers when Origin header is absent', done => { + const request = buildRequest({ + apiMethod: 'bucketGet', + httpMethod: 'GET', + url: '/', + query: {}, + }); + delete request.headers.origin; + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); - api.callApiMethod('bucketGet', request, response, log, err => { - assert(err && err.is.AccessDenied); - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined); - done(); - }); + api.callApiMethod('bucketGet', request, response, log, err => { + assert(err && err.is.AccessDenied); + assert.strictEqual(response.getHeader('access-control-allow-origin'), undefined); + done(); }); + }); + + it('does not attach CORS headers when origin does not match any rule', done => { + const request = buildRequest({ + apiMethod: 'bucketGet', + httpMethod: 'GET', + url: '/', + query: {}, + }); + request.headers.origin = 'http://not-allowed.test'; + // The bucket's CORS config allows any method from foo.test + // plus GET from *. Use PUT from a different origin so neither + // condition matches. + request.method = 'PUT'; + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); + + api.callApiMethod('bucketGet', request, response, log, err => { + assert(err && err.is.AccessDenied); + assert.strictEqual(response.getHeader('access-control-allow-origin'), undefined); + done(); + }); + }); }); describe('CORS headers on 403 via handler (fast path)', () => { @@ -228,67 +212,63 @@ describe('CORS headers on 403 via handler (fast path)', () => { // denies at its own ACL check (bucket is owned by accessKey1). const otherAuth = makeAuthInfo('accessKey2'); const authServer = { - doAuth: sandbox.stub().callsArgWith(2, null, otherAuth, - [{ isAllowed: true, isImplicit: false }], null, {}), + doAuth: sandbox.stub().callsArgWith(2, null, otherAuth, [{ isAllowed: true, isImplicit: false }], null, {}), }; sandbox.stub(auth, 'server').value(authServer); }); afterEach(() => sandbox.restore()); - it('forwards handler-provided corsHeaders without setting headers ' - + 'on the response directly', done => { + it('forwards handler-provided corsHeaders without setting headers ' + 'on the response directly', done => { const request = buildRequest({ - apiMethod: 'bucketGet', httpMethod: 'GET', - url: '/', query: {}, + apiMethod: 'bucketGet', + httpMethod: 'GET', + url: '/', + query: {}, }); const response = buildResponseSpy(sandbox); const log = buildLog(sandbox); - api.callApiMethod('bucketGet', request, response, log, - (err, xml, corsHeaders) => { - assert(err, 'expected an error'); - assert(err.is && err.is.AccessDenied, - `expected AccessDenied, got ${err.code}`); - assert(corsHeaders, - 'handler should have supplied corsHeaders'); - assert.strictEqual( - corsHeaders['access-control-allow-origin'], origin); - // Fast path: wrapper forwards corsHeaders via the callback - // instead of setting them on the response directly. - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined); - done(); - }); + api.callApiMethod('bucketGet', request, response, log, (err, xml, corsHeaders) => { + assert(err, 'expected an error'); + assert(err.is && err.is.AccessDenied, `expected AccessDenied, got ${err.code}`); + assert(corsHeaders, 'handler should have supplied corsHeaders'); + assert.strictEqual(corsHeaders['access-control-allow-origin'], origin); + // Fast path: wrapper forwards corsHeaders via the callback + // instead of setting them on the response directly. + assert.strictEqual(response.getHeader('access-control-allow-origin'), undefined); + done(); + }); }); - it('makes at most 2 metadata.getBucket calls on the error path', - done => { - const getBucketSpy = sandbox.spy(metadata, 'getBucket'); - const request = buildRequest({ - apiMethod: 'bucketHead', httpMethod: 'HEAD', - url: '/', query: {}, - }); - // Origin that matches no CORS rule -> handler emits empty - // corsHeaders -> fast path misses -> wrapper falls back to a - // second getBucket. The handler's own call (1) + the wrapper - // fallback (1) is the documented ceiling - see the comment - // on wrapCallbackWithErrorCorsHeaders in lib/api/api.js. Use - // <= so this remains a future-proof ceiling: optimizations - // that reduce the count are welcome. - request.headers.origin = 'http://not-allowed.test'; - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); + it('makes at most 2 metadata.getBucket calls on the error path', done => { + const getBucketSpy = sandbox.spy(metadata, 'getBucket'); + const request = buildRequest({ + apiMethod: 'bucketHead', + httpMethod: 'HEAD', + url: '/', + query: {}, + }); + // Origin that matches no CORS rule -> handler emits empty + // corsHeaders -> fast path misses -> wrapper falls back to a + // second getBucket. The handler's own call (1) + the wrapper + // fallback (1) is the documented ceiling - see the comment + // on wrapCallbackWithErrorCorsHeaders in lib/api/api.js. Use + // <= so this remains a future-proof ceiling: optimizations + // that reduce the count are welcome. + request.headers.origin = 'http://not-allowed.test'; + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); - api.callApiMethod('bucketHead', request, response, log, err => { - assert(err && err.is.AccessDenied); - assert(getBucketSpy.callCount <= 2, - 'expected at most 2 metadata.getBucket calls, got ' - + `${getBucketSpy.callCount}`); - done(); - }); + api.callApiMethod('bucketHead', request, response, log, err => { + assert(err && err.is.AccessDenied); + assert( + getBucketSpy.callCount <= 2, + 'expected at most 2 metadata.getBucket calls, got ' + `${getBucketSpy.callCount}`, + ); + done(); }); + }); }); describe('CORS headers on copy operations', () => { @@ -331,31 +311,40 @@ describe('CORS headers on copy operations', () => { allowedMethods: ['PUT'], allowedOrigins: [reqOrigin], }); - async.series([ - cb => bucketPut(authInfo, destPutReq, log, cb), - cb => bucketPut(authInfo, srcPutReq, log, cb), - cb => bucketPutCors(authInfo, - destCors.createBucketCorsRequest('PUT', destBucket), log, cb), - cb => bucketPutCors(authInfo, - srcCors.createBucketCorsRequest('PUT', srcBucket), log, cb), - ], done); + async.series( + [ + cb => bucketPut(authInfo, destPutReq, log, cb), + cb => bucketPut(authInfo, srcPutReq, log, cb), + cb => bucketPutCors(authInfo, destCors.createBucketCorsRequest('PUT', destBucket), log, cb), + cb => bucketPutCors(authInfo, srcCors.createBucketCorsRequest('PUT', srcBucket), log, cb), + ], + done, + ); }); beforeEach(() => { sandbox = sinon.createSandbox(); const authServer = { - doAuth: sandbox.stub().callsArgWith(2, null, authInfo, - [{ isAllowed: true, isImplicit: false }, - { isAllowed: true, isImplicit: false }], null, {}), + doAuth: sandbox.stub().callsArgWith( + 2, + null, + authInfo, + [ + { isAllowed: true, isImplicit: false }, + { isAllowed: true, isImplicit: false }, + ], + null, + {}, + ), }; sandbox.stub(auth, 'server').value(authServer); }); afterEach(() => sandbox.restore()); - it('does not leak source-bucket CORS headers on objectCopy errors', - done => { - const request = new DummyRequest({ + it('does not leak source-bucket CORS headers on objectCopy errors', done => { + const request = new DummyRequest( + { bucketName: destBucket, objectKey: 'destkey', headers: { @@ -366,23 +355,26 @@ describe('CORS headers on copy operations', () => { url: `/${destBucket}/destkey`, query: {}, method: 'PUT', - }, Buffer.alloc(0)); - const response = buildResponseSpy(sandbox); - const log = buildLog(sandbox); + }, + Buffer.alloc(0), + ); + const response = buildResponseSpy(sandbox); + const log = buildLog(sandbox); - api.callApiMethod('objectCopy', request, response, log, err => { - assert(err, 'expected an error'); - // Dest does not allow PUT from reqOrigin, so no CORS - // headers should be set. If the wrapper used the source - // bucket (which DOES allow PUT from reqOrigin) we would - // see access-control-allow-origin: http://foo.test here. - assert.strictEqual( - response.getHeader('access-control-allow-origin'), - undefined, - 'wrapper must not apply source-bucket CORS headers'); - done(); - }); + api.callApiMethod('objectCopy', request, response, log, err => { + assert(err, 'expected an error'); + // Dest does not allow PUT from reqOrigin, so no CORS + // headers should be set. If the wrapper used the source + // bucket (which DOES allow PUT from reqOrigin) we would + // see access-control-allow-origin: http://foo.test here. + assert.strictEqual( + response.getHeader('access-control-allow-origin'), + undefined, + 'wrapper must not apply source-bucket CORS headers', + ); + done(); }); + }); }); describe('CORS headers on 200 successful responses (per-handler)', () => { @@ -403,10 +395,8 @@ describe('CORS headers on 200 successful responses (per-handler)', () => { }; bucketGet(authInfo, request, log, (err, xml, corsHeaders) => { assert.ifError(err); - assert(corsHeaders, - 'expected corsHeaders to be set on successful bucketGet'); - assert(corsHeaders['access-control-allow-origin'], - 'expected access-control-allow-origin on 200'); + assert(corsHeaders, 'expected corsHeaders to be set on successful bucketGet'); + assert(corsHeaders['access-control-allow-origin'], 'expected access-control-allow-origin on 200'); done(); }); }); @@ -426,9 +416,10 @@ describe('CORS headers on 200 successful responses (per-handler)', () => { }; bucketGetCors(authInfo, request, log, (err, xml, corsHeaders) => { assert.ifError(err); - assert(corsHeaders - && corsHeaders['access-control-allow-origin'], - 'expected access-control-allow-origin on 200'); + assert( + corsHeaders && corsHeaders['access-control-allow-origin'], + 'expected access-control-allow-origin on 200', + ); done(); }); }); diff --git a/tests/unit/api/deleteMarker.js b/tests/unit/api/deleteMarker.js index a5610c057e..1425990094 100644 --- a/tests/unit/api/deleteMarker.js +++ b/tests/unit/api/deleteMarker.js @@ -48,10 +48,11 @@ function _createBucketPutVersioningReq(status) { query: { versioning: '' }, actionImplicitDenies: false, }; - const xml = '' + - `${status}` + - ''; + const xml = + '' + + `${status}` + + ''; request.post = xml; return request; } @@ -79,8 +80,7 @@ function _createMultiObjectDeleteRequest(numObjects) { } xml.push(''); request.post = xml.join(''); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); return request; } @@ -94,12 +94,7 @@ const expectedAcl = { READ_ACP: [], }; -const undefHeadersExpected = [ - 'cache-control', - 'content-disposition', - 'content-encoding', - 'expires', -]; +const undefHeadersExpected = ['cache-control', 'content-disposition', 'content-encoding', 'expires']; describe('delete marker creation', () => { beforeEach(done => { @@ -108,8 +103,7 @@ describe('delete marker creation', () => { if (err) { return done(err); } - return bucketPutVersioning(authInfo, enableVersioningRequest, - log, done); + return bucketPutVersioning(authInfo, enableVersioningRequest, log, done); }); }); @@ -119,46 +113,43 @@ describe('delete marker creation', () => { function _assertDeleteMarkerMd(deleteResultVersionId, isLatest, callback) { const options = { - versionId: isLatest ? undefined : - versionIdUtils.decode(deleteResultVersionId), + versionId: isLatest ? undefined : versionIdUtils.decode(deleteResultVersionId), }; - return metadata.getObjectMD(bucketName, objectName, options, log, - (err, deleteMarkerMD) => { - assert.strictEqual(err, null); - const mdVersionId = deleteMarkerMD.versionId; - assert.strictEqual(deleteMarkerMD.isDeleteMarker, true); - assert.strictEqual( - versionIdUtils.encode(mdVersionId), - deleteResultVersionId); - assert.strictEqual(deleteMarkerMD['content-length'], 0); - assert.strictEqual(deleteMarkerMD.location, null); - assert.deepStrictEqual(deleteMarkerMD.acl, expectedAcl); - undefHeadersExpected.forEach(header => { - assert.strictEqual(deleteMarkerMD[header], undefined); - }); - return callback(); + return metadata.getObjectMD(bucketName, objectName, options, log, (err, deleteMarkerMD) => { + assert.strictEqual(err, null); + const mdVersionId = deleteMarkerMD.versionId; + assert.strictEqual(deleteMarkerMD.isDeleteMarker, true); + assert.strictEqual(versionIdUtils.encode(mdVersionId), deleteResultVersionId); + assert.strictEqual(deleteMarkerMD['content-length'], 0); + assert.strictEqual(deleteMarkerMD.location, null); + assert.deepStrictEqual(deleteMarkerMD.acl, expectedAcl); + undefHeadersExpected.forEach(header => { + assert.strictEqual(deleteMarkerMD[header], undefined); }); + return callback(); + }); } - it('should create a delete marker if versioning enabled and deleting ' + - 'object without specifying version id', done => { - objectDelete(authInfo, testDeleteRequest, log, (err, delResHeaders) => { - if (err) { - return done(err); - } - assert.strictEqual(delResHeaders['x-amz-delete-marker'], true); - assert(delResHeaders['x-amz-version-id']); - return _assertDeleteMarkerMd(delResHeaders['x-amz-version-id'], - true, done); - }); - }); + it( + 'should create a delete marker if versioning enabled and deleting ' + 'object without specifying version id', + done => { + objectDelete(authInfo, testDeleteRequest, log, (err, delResHeaders) => { + if (err) { + return done(err); + } + assert.strictEqual(delResHeaders['x-amz-delete-marker'], true); + assert(delResHeaders['x-amz-version-id']); + return _assertDeleteMarkerMd(delResHeaders['x-amz-version-id'], true, done); + }); + }, + ); - it('multi-object delete should create delete markers if versioning ' + - 'enabled and items do not have version id specified', done => { - const testMultiObjectDeleteRequest = - _createMultiObjectDeleteRequest(3); - return multiObjectDelete(authInfo, testMultiObjectDeleteRequest, log, - (err, xml) => { + it( + 'multi-object delete should create delete markers if versioning ' + + 'enabled and items do not have version id specified', + done => { + const testMultiObjectDeleteRequest = _createMultiObjectDeleteRequest(3); + return multiObjectDelete(authInfo, testMultiObjectDeleteRequest, log, (err, xml) => { if (err) { return done(err); } @@ -167,14 +158,18 @@ describe('delete marker creation', () => { return done(err); } const results = parsedResult.DeleteResult.Deleted; - return async.forEach(results, (result, cb) => { - assert.strictEqual(result.Key[0], objectName); - assert.strictEqual(result.DeleteMarker[0], 'true'); - assert(result.DeleteMarkerVersionId[0]); - _assertDeleteMarkerMd(result.DeleteMarkerVersionId[0], - false, cb); - }, err => done(err)); + return async.forEach( + results, + (result, cb) => { + assert.strictEqual(result.Key[0], objectName); + assert.strictEqual(result.DeleteMarker[0], 'true'); + assert(result.DeleteMarkerVersionId[0]); + _assertDeleteMarkerMd(result.DeleteMarkerVersionId[0], false, cb); + }, + err => done(err), + ); }); }); - }); + }, + ); }); diff --git a/tests/unit/api/deletedFlagBucket.js b/tests/unit/api/deletedFlagBucket.js index 83d699f4e2..1d3198e795 100644 --- a/tests/unit/api/deletedFlagBucket.js +++ b/tests/unit/api/deletedFlagBucket.js @@ -14,18 +14,12 @@ const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketDelete = require('../../../lib/api/bucketDelete'); const bucketDeleteCors = require('../../../lib/api/bucketDeleteCors'); const bucketDeleteWebsite = require('../../../lib/api/bucketDeleteWebsite'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const { config } = require('../../../lib/Config'); const constants = require('../../../constants'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); -const { cleanup, - createAlteredRequest, - DummyRequestLogger, - makeAuthInfo } - = require('../helpers'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const { cleanup, createAlteredRequest, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); const listParts = require('../../../lib/api/listParts'); const metadata = require('../metadataswitch'); @@ -69,21 +63,16 @@ const serviceGetRequest = { const userBucketOwner = 'admin'; const creationDate = new Date().toJSON(); -const usersBucket = new BucketInfo(usersBucketName, - userBucketOwner, userBucketOwner, creationDate); - +const usersBucket = new BucketInfo(usersBucketName, userBucketOwner, userBucketOwner, creationDate); function checkBucketListing(authInfo, bucketName, expectedListingLength, done) { return serviceGet(authInfo, serviceGetRequest, log, (err, data) => { parseString(data, (err, result) => { if (expectedListingLength > 0) { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket.length, expectedListingLength); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[0].Name[0], bucketName); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket.length, expectedListingLength); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[0].Name[0], bucketName); } else { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].length, 0); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].length, 0); } done(); }); @@ -105,15 +94,12 @@ function confirmDeleted(done) { }); } - describe('deleted flag bucket handling', () => { beforeEach(done => { cleanup(); - const bucketMD = new BucketInfo(bucketName, canonicalID, - authInfo.getAccountDisplayName(), creationDate); + const bucketMD = new BucketInfo(bucketName, canonicalID, authInfo.getAccountDisplayName(), creationDate); bucketMD.addDeletedFlag(); - bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), - 'FULL_CONTROL'); + bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), 'FULL_CONTROL'); bucketMD.setLocationConstraint(locationConstraint); metadata.createBucket(bucketName, bucketMD, log, () => { metadata.createBucket(usersBucketName, usersBucket, log, () => { @@ -122,77 +108,103 @@ describe('deleted flag bucket handling', () => { }); }); - it('putBucket request should recreate bucket with deleted flag if ' + - 'request is from same account that originally put', done => { - bucketPut(authInfo, baseTestRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._deleted, false); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(authInfo, bucketName, 1, done); + it( + 'putBucket request should recreate bucket with deleted flag if ' + + 'request is from same account that originally put', + done => { + bucketPut(authInfo, baseTestRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._deleted, false); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); - }); - }); + }, + ); - it('putBucket request should return error if ' + - 'different account sends put bucket request for bucket with ' + - 'deleted flag', done => { - bucketPut(otherAccountAuthInfo, baseTestRequest, log, err => { - assert.strictEqual(err.is.BucketAlreadyExists, true); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._deleted, true); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(otherAccountAuthInfo, - bucketName, 0, done); + it( + 'putBucket request should return error if ' + + 'different account sends put bucket request for bucket with ' + + 'deleted flag', + done => { + bucketPut(otherAccountAuthInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.BucketAlreadyExists, true); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._deleted, true); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(otherAccountAuthInfo, bucketName, 0, done); + }); }); - }); - }); + }, + ); - it('ACLs from new putBucket request should overwrite ACLs saved ' + - 'in metadata of bucket with deleted flag', done => { - const alteredRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPut(authInfo, alteredRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._deleted, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(authInfo, bucketName, 1, done); + it( + 'ACLs from new putBucket request should overwrite ACLs saved ' + 'in metadata of bucket with deleted flag', + done => { + const alteredRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPut(authInfo, alteredRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._deleted, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); - }); - }); + }, + ); - it('putBucketACL request should recreate bucket with deleted flag if ' + - 'request is from same account that originally put', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - putACLRequest.query = { acl: '' }; - bucketPutACL(authInfo, putACLRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - return checkBucketListing(authInfo, bucketName, 1, done); + it( + 'putBucketACL request should recreate bucket with deleted flag if ' + + 'request is from same account that originally put', + done => { + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + putACLRequest.query = { acl: '' }; + bucketPutACL(authInfo, putACLRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); - }); - }); + }, + ); - it('putBucketACL request on bucket with deleted flag should return ' + - 'NoSuchBucket error if request is from another authorized account', + it( + 'putBucketACL request on bucket with deleted flag should return ' + + 'NoSuchBucket error if request is from another authorized account', // Do not want different account recreating a bucket that the bucket // owner wanted deleted even if the other account is authorized to // change the ACLs done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); bucketPutACL(otherAccountAuthInfo, putACLRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); metadata.getBucket(bucketName, log, (err, data) => { @@ -203,14 +215,21 @@ describe('deleted flag bucket handling', () => { done(); }); }); - }); + }, + ); - it('putBucketACL request on bucket with deleted flag should return ' + - 'AccessDenied error if request is from unauthorized account', + it( + 'putBucketACL request on bucket with deleted flag should return ' + + 'AccessDenied error if request is from unauthorized account', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); const unauthorizedAccount = makeAuthInfo('keepMeOut'); bucketPutACL(unauthorizedAccount, putACLRequest, log, err => { assert.strictEqual(err.is.AccessDenied, true); @@ -222,7 +241,8 @@ describe('deleted flag bucket handling', () => { done(); }); }); - }); + }, + ); describe('objectPut on a bucket with deleted flag', () => { const objName = 'objectName'; @@ -232,10 +252,8 @@ describe('deleted flag bucket handling', () => { }); }); - it('objectPut request from account that originally created ' + - 'should recreate bucket', done => { - const setUpRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('objectPut request from account that originally created ' + 'should recreate bucket', done => { + const setUpRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); setUpRequest.objectKey = objName; const postBody = Buffer.from('I am a body', 'utf8'); const md5Hash = crypto.createHash('md5'); @@ -247,46 +265,43 @@ describe('deleted flag bucket handling', () => { assert.strictEqual(data._transient, false); assert.strictEqual(data._deleted, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.getObjectMD(bucketName, objName, {}, log, - (err, obj) => { - assert.ifError(err); - assert.strictEqual(obj['content-md5'], etag); - return checkBucketListing(authInfo, - bucketName, 1, done); - }); + metadata.getObjectMD(bucketName, objName, {}, log, (err, obj) => { + assert.ifError(err); + assert.strictEqual(obj['content-md5'], etag); + return checkBucketListing(authInfo, bucketName, 1, done); + }); }); }); }); }); - it('should return NoSuchBucket error on an objectPut request from ' + - 'different account when there is a deleted flag', done => { - const setUpRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - setUpRequest.objectKey = 'objectName'; - const postBody = Buffer.from('I am a body', 'utf8'); - const putObjRequest = new DummyRequest(setUpRequest, postBody); - objectPut(otherAccountAuthInfo, putObjRequest, undefined, log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); - }); + it( + 'should return NoSuchBucket error on an objectPut request from ' + + 'different account when there is a deleted flag', + done => { + const setUpRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + setUpRequest.objectKey = 'objectName'; + const postBody = Buffer.from('I am a body', 'utf8'); + const putObjRequest = new DummyRequest(setUpRequest, postBody); + objectPut(otherAccountAuthInfo, putObjRequest, undefined, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); + }, + ); describe('initiateMultipartUpload on a bucket with deleted flag', () => { const objName = 'objectName'; after(done => { - metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + - `${bucketName}`, objName, {}, log, () => { - metadata.deleteBucket(`${constants.mpuBucketPrefix}` + - `${bucketName}`, log, () => { - done(); - }); + metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + `${bucketName}`, objName, {}, log, () => { + metadata.deleteBucket(`${constants.mpuBucketPrefix}` + `${bucketName}`, log, () => { + done(); }); + }); }); it('should recreate bucket with deleted flag', done => { - const initiateRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + const initiateRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); initiateRequest.objectKey = objName; initiateRequest.url = `/${objName}?uploads`; initiateMultipartUpload(authInfo, initiateRequest, log, err => { @@ -295,185 +310,196 @@ describe('deleted flag bucket handling', () => { assert.strictEqual(data._transient, false); assert.strictEqual(data._deleted, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.listObject(`${constants.mpuBucketPrefix}` + - `${bucketName}`, + metadata.listObject( + `${constants.mpuBucketPrefix}` + `${bucketName}`, { prefix: `overview${constants.splitter}${objName}` }, - log, (err, results) => { + log, + (err, results) => { assert.ifError(err); assert.strictEqual(results.Contents.length, 1); done(); - }); + }, + ); }); }); }); }); - it('should return NoSuchBucket error on an initiateMultipartUpload ' + - 'request from different account when there is a deleted flag', done => { - const initiateRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - initiateRequest.objectKey = 'objectName'; - initiateMultipartUpload(otherAccountAuthInfo, initiateRequest, log, - err => { + it( + 'should return NoSuchBucket error on an initiateMultipartUpload ' + + 'request from different account when there is a deleted flag', + done => { + const initiateRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + initiateRequest.objectKey = 'objectName'; + initiateMultipartUpload(otherAccountAuthInfo, initiateRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); - }); + }, + ); - it('deleteBucket request should complete deletion ' + - 'of bucket with deleted flag', done => { + it('deleteBucket request should complete deletion ' + 'of bucket with deleted flag', done => { bucketDelete(authInfo, baseTestRequest, log, err => { assert.ifError(err); confirmDeleted(done); }); }); - it('deleteBucket request should return error if account not ' + - 'authorized', done => { - bucketDelete(otherAccountAuthInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); + it('deleteBucket request should return error if account not ' + 'authorized', done => { + bucketDelete(otherAccountAuthInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); + }); }); - it('bucketDeleteWebsite request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketDeleteWebsite(authInfo, baseTestRequest, - log, err => { + it( + 'bucketDeleteWebsite request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + bucketDeleteWebsite(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('bucketGet request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGet(authInfo, baseTestRequest, - log, err => { + it( + 'bucketGet request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketGet(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('bucketGetACL request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGetACL(authInfo, baseTestRequest, - log, err => { + it( + 'bucketGetACL request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketGetACL(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('bucketGetCors request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGetCors(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketGetCors request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketGetCors(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketPutCors request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - const bucketPutCorsRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutCorsRequest.post = '' + - 'PUT' + - 'http://www.example.com' + - ''; - bucketPutCorsRequest.headers['content-md5'] = crypto.createHash('md5') - .update(bucketPutCorsRequest.post, 'utf8').digest('base64'); - bucketPutCors(authInfo, bucketPutCorsRequest, log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketPutCors request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + const bucketPutCorsRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + bucketPutCorsRequest.post = + '' + + 'PUT' + + 'http://www.example.com' + + ''; + bucketPutCorsRequest.headers['content-md5'] = crypto + .createHash('md5') + .update(bucketPutCorsRequest.post, 'utf8') + .digest('base64'); + bucketPutCors(authInfo, bucketPutCorsRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketDeleteCors request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketDeleteCors(authInfo, baseTestRequest, log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketDeleteCors request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + bucketDeleteCors(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketGetWebsite request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketGetWebsite(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketGetWebsite request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + bucketGetWebsite(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketPutWebsite request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - const bucketPutWebsiteRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutWebsiteRequest.post = '' + - 'index.html' + - ''; - bucketPutWebsite(authInfo, bucketPutWebsiteRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'bucketPutWebsite request on bucket with delete flag should return ' + + 'NoSuchBucket error and complete deletion', + done => { + const bucketPutWebsiteRequest = createAlteredRequest( + {}, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPutWebsiteRequest.post = + '' + + 'index.html' + + ''; + bucketPutWebsite(authInfo, bucketPutWebsiteRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('bucketHead request on bucket with delete flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - bucketHead(authInfo, baseTestRequest, - log, err => { + it( + 'bucketHead request on bucket with delete flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + bucketHead(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - function checkForNoSuchUploadError(apiAction, partNumber, done, - extraArgNeeded) { - const mpuRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + function checkForNoSuchUploadError(apiAction, partNumber, done, extraArgNeeded) { + const mpuRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; mpuRequest.objectKey = 'objectName'; mpuRequest.query = { uploadId, partNumber }; if (extraArgNeeded) { - return apiAction(authInfo, mpuRequest, undefined, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - return done(); - }); - } - return apiAction(authInfo, mpuRequest, - log, err => { + return apiAction(authInfo, mpuRequest, undefined, log, err => { assert.strictEqual(err.is.NoSuchUpload, true); return done(); }); + } + return apiAction(authInfo, mpuRequest, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + return done(); + }); } - it('completeMultipartUpload request on bucket with deleted flag should ' + - 'return NoSuchUpload error', done => { + it('completeMultipartUpload request on bucket with deleted flag should ' + 'return NoSuchUpload error', done => { checkForNoSuchUploadError(completeMultipartUpload, null, done); }); - it('listParts request on bucket with deleted flag should ' + - 'return NoSuchUpload error', done => { + it('listParts request on bucket with deleted flag should ' + 'return NoSuchUpload error', done => { checkForNoSuchUploadError(listParts, null, done); }); describe('multipartDelete request on a bucket with deleted flag', () => { - it('should return NoSuchUpload error if legacyAWSBehavior is enabled', - done => { - config.locationConstraints[locationConstraint]. - legacyAwsBehavior = true; + it('should return NoSuchUpload error if legacyAWSBehavior is enabled', done => { + config.locationConstraints[locationConstraint].legacyAwsBehavior = true; checkForNoSuchUploadError(multipartDelete, null, done); }); - it('should return no error if legacyAWSBehavior is not enabled', - done => { - config.locationConstraints[locationConstraint]. - legacyAwsBehavior = false; - const mpuRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('should return no error if legacyAWSBehavior is not enabled', done => { + config.locationConstraints[locationConstraint].legacyAwsBehavior = false; + const mpuRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; mpuRequest.objectKey = 'objectName'; mpuRequest.query = { uploadId }; @@ -484,64 +510,61 @@ describe('deleted flag bucket handling', () => { }); }); - it('objectPutPart request on bucket with deleted flag should ' + - 'return NoSuchUpload error', done => { + it('objectPutPart request on bucket with deleted flag should ' + 'return NoSuchUpload error', done => { checkForNoSuchUploadError(objectPutPart, '1', done, true); }); - it('list multipartUploads request on bucket with deleted flag should ' + - 'return NoSuchBucket error', done => { - const listRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('list multipartUploads request on bucket with deleted flag should ' + 'return NoSuchBucket error', done => { + const listRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); listRequest.query = {}; - listMultipartUploads(authInfo, listRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + listMultipartUploads(authInfo, listRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('objectGet request on bucket with deleted flag should' + - 'return NoSuchBucket error and finish deletion', + it( + 'objectGet request on bucket with deleted flag should' + 'return NoSuchBucket error and finish deletion', done => { - objectGet(authInfo, baseTestRequest, false, - log, err => { + objectGet(authInfo, baseTestRequest, false, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); - }); + }, + ); - it('objectGetACL request on bucket with deleted flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - objectGetACL(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'objectGetACL request on bucket with deleted flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + objectGetACL(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('objectHead request on bucket with deleted flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - objectHead(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'objectHead request on bucket with deleted flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + objectHead(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('objectPutACL request on bucket with deleted flag should return ' + - 'NoSuchBucket error and complete deletion', done => { - objectPutACL(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - confirmDeleted(done); - }); - }); + it( + 'objectPutACL request on bucket with deleted flag should return ' + 'NoSuchBucket error and complete deletion', + done => { + objectPutACL(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + confirmDeleted(done); + }); + }, + ); - it('objectDelete request on bucket with deleted flag should return ' + - 'NoSuchBucket error', done => { - objectDelete(authInfo, baseTestRequest, - log, err => { + it('objectDelete request on bucket with deleted flag should return ' + 'NoSuchBucket error', done => { + objectDelete(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); confirmDeleted(done); }); diff --git a/tests/unit/api/listMultipartUploads.js b/tests/unit/api/listMultipartUploads.js index 3141ab0108..e7988821ef 100644 --- a/tests/unit/api/listMultipartUploads.js +++ b/tests/unit/api/listMultipartUploads.js @@ -58,40 +58,39 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - it('should return the name of the common prefix ' + - 'of common prefix object keys for multipart uploads if delimiter ' + - 'and prefix specified', done => { - const commonPrefix = `${prefix}${delimiter}`; - const testListRequest = { - bucketName, - namespace, - headers: { host: '/' }, - url: `/${bucketName}?uploads&delimiter=/&prefix=sub`, - query: { delimiter, prefix }, - actionImplicitDenies: false, - }; + it( + 'should return the name of the common prefix ' + + 'of common prefix object keys for multipart uploads if delimiter ' + + 'and prefix specified', + done => { + const commonPrefix = `${prefix}${delimiter}`; + const testListRequest = { + bucketName, + namespace, + headers: { host: '/' }, + url: `/${bucketName}?uploads&delimiter=/&prefix=sub`, + query: { delimiter, prefix }, + actionImplicitDenies: false, + }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .CommonPrefixes[0].Prefix[0], - commonPrefix); - done(); - }); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => + initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.CommonPrefixes[0].Prefix[0], commonPrefix); + done(); + }, + ); + }, + ); - it('should return list of all multipart uploads if ' + - 'no delimiter specified', done => { + it('should return list of all multipart uploads if ' + 'no delimiter specified', done => { const testListRequest = { bucketName, namespace, @@ -101,31 +100,24 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => - listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], objectName1); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1].Key[0], objectName2); - assert.strictEqual(result.ListMultipartUploadsResult - .IsTruncated[0], 'false'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName1); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1].Key[0], objectName2); + assert.strictEqual(result.ListMultipartUploadsResult.IsTruncated[0], 'false'); + done(); + }, + ); }); - it('should return no more keys than ' + - 'max-uploads specified', done => { + it('should return no more keys than ' + 'max-uploads specified', done => { const testListRequest = { bucketName, namespace, @@ -135,34 +127,26 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], objectName1); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1], undefined); - assert.strictEqual(result.ListMultipartUploadsResult - .IsTruncated[0], 'true'); - assert.strictEqual(result.ListMultipartUploadsResult - .NextKeyMarker[0], objectName1); - assert(result.ListMultipartUploadsResult - .NextUploadIdMarker[0].length > 5); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName1); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined); + assert.strictEqual(result.ListMultipartUploadsResult.IsTruncated[0], 'true'); + assert.strictEqual(result.ListMultipartUploadsResult.NextKeyMarker[0], objectName1); + assert(result.ListMultipartUploadsResult.NextUploadIdMarker[0].length > 5); + done(); + }, + ); }); - it('should url encode object key name ' + - 'if requested', done => { + it('should url encode object key name ' + 'if requested', done => { const testListRequest = { bucketName, namespace, @@ -172,26 +156,21 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, - testInitiateMPURequest3, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, - testListRequest, log, next), - (result, corsHeaders, next) => - parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[0].Key[0], querystring.escape(objectName3)); - assert.strictEqual(result.ListMultipartUploadsResult - .Upload[1].Key[0], querystring.escape(objectName1)); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], querystring.escape(objectName3)); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1].Key[0], querystring.escape(objectName1)); + done(); + }, + ); }); it('should return key following specified key-marker', done => { @@ -204,19 +183,21 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => parseString(result, corsHeaders, next), - ], - (err, result) => { - assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName2); - assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest2, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest3, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + assert.strictEqual(result.ListMultipartUploadsResult.Upload[0].Key[0], objectName2); + assert.strictEqual(result.ListMultipartUploadsResult.Upload[1], undefined); + done(); + }, + ); }); it('should include ChecksumAlgorithm and ChecksumType when set on MPU', done => { @@ -238,19 +219,21 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => parseString(result, corsHeaders, next), - ], - (err, result) => { - const upload = result.ListMultipartUploadsResult.Upload[0]; - assert.strictEqual(upload.Key[0], checksumKey); - assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32'); - assert.strictEqual(upload.ChecksumType[0], 'COMPOSITE'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const upload = result.ListMultipartUploadsResult.Upload[0]; + assert.strictEqual(upload.Key[0], checksumKey); + assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32'); + assert.strictEqual(upload.ChecksumType[0], 'COMPOSITE'); + done(); + }, + ); }); it('should not include ChecksumAlgorithm or ChecksumType when not set on MPU', done => { @@ -263,19 +246,21 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => parseString(result, corsHeaders, next), - ], - (err, result) => { - const upload = result.ListMultipartUploadsResult.Upload[0]; - assert.strictEqual(upload.Key[0], objectName1); - assert.strictEqual(upload.ChecksumAlgorithm, undefined); - assert.strictEqual(upload.ChecksumType, undefined); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitiateMPURequest1, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const upload = result.ListMultipartUploadsResult.Upload[0]; + assert.strictEqual(upload.Key[0], objectName1); + assert.strictEqual(upload.ChecksumAlgorithm, undefined); + assert.strictEqual(upload.ChecksumType, undefined); + done(); + }, + ); }); it('should include ChecksumAlgorithm and ChecksumType with explicit type', done => { @@ -300,19 +285,21 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => parseString(result, corsHeaders, next), - ], - (err, result) => { - const upload = result.ListMultipartUploadsResult.Upload[0]; - assert.strictEqual(upload.Key[0], checksumKey); - assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32'); - assert.strictEqual(upload.ChecksumType[0], 'FULL_OBJECT'); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const upload = result.ListMultipartUploadsResult.Upload[0]; + assert.strictEqual(upload.Key[0], checksumKey); + assert.strictEqual(upload.ChecksumAlgorithm[0], 'CRC32'); + assert.strictEqual(upload.ChecksumType[0], 'FULL_OBJECT'); + done(); + }, + ); }); it('should list mixed uploads with and without checksum correctly', done => { @@ -343,25 +330,27 @@ describe('listMultipartUploads API', () => { actionImplicitDenies: false, }; - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), - (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitNoChecksumRequest, log, next), - (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), - (result, corsHeaders, next) => parseString(result, corsHeaders, next), - ], - (err, result) => { - const uploads = result.ListMultipartUploadsResult.Upload; - assert.strictEqual(uploads.length, 2); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, testInitChecksumRequest, log, next), + (result, corsHeaders, next) => initiateMultipartUpload(authInfo, testInitNoChecksumRequest, log, next), + (result, corsHeaders, next) => listMultipartUploads(authInfo, testListRequest, log, next), + (result, corsHeaders, next) => parseString(result, corsHeaders, next), + ], + (err, result) => { + const uploads = result.ListMultipartUploadsResult.Upload; + assert.strictEqual(uploads.length, 2); - const withChecksum = uploads.find(u => u.Key[0] === checksumKey); - assert.strictEqual(withChecksum.ChecksumAlgorithm[0], 'SHA256'); - assert.strictEqual(withChecksum.ChecksumType[0], 'COMPOSITE'); + const withChecksum = uploads.find(u => u.Key[0] === checksumKey); + assert.strictEqual(withChecksum.ChecksumAlgorithm[0], 'SHA256'); + assert.strictEqual(withChecksum.ChecksumType[0], 'COMPOSITE'); - const withoutChecksum = uploads.find(u => u.Key[0] === noChecksumKey); - assert.strictEqual(withoutChecksum.ChecksumAlgorithm, undefined); - assert.strictEqual(withoutChecksum.ChecksumType, undefined); - done(); - }); + const withoutChecksum = uploads.find(u => u.Key[0] === noChecksumKey); + assert.strictEqual(withoutChecksum.ChecksumAlgorithm, undefined); + assert.strictEqual(withoutChecksum.ChecksumType, undefined); + done(); + }, + ); }); }); diff --git a/tests/unit/api/listParts.js b/tests/unit/api/listParts.js index edcab83cd3..cbd408cec0 100644 --- a/tests/unit/api/listParts.js +++ b/tests/unit/api/listParts.js @@ -24,16 +24,13 @@ const mpuBucket = `${constants.mpuBucketPrefix}${bucketName}`; const uploadKey = '$makememulti'; const sixMBObjectETag = '"f3a9fb2071d3503b703938a74eb99846"'; const lastPieceETag = '"555e4cd2f9eff38109d7a3ab13995a32"'; -const overviewKey = `overview${splitter}$makememulti${splitter}4db92ccc-` + - 'd89d-49d3-9fa6-e9c2c1eb31b0'; +const overviewKey = `overview${splitter}$makememulti${splitter}4db92ccc-` + 'd89d-49d3-9fa6-e9c2c1eb31b0'; const partOneKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00001`; -const partTwoKey = '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0' + - `${splitter}00002`; +const partTwoKey = '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0' + `${splitter}00002`; const partThreeKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00003`; const partFourKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00004`; const partFiveKey = `4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0${splitter}00005`; -const partKeys = [partOneKey, partTwoKey, partThreeKey, partFourKey, - partFiveKey]; +const partKeys = [partOneKey, partTwoKey, partThreeKey, partFourKey, partFiveKey]; const partChecksumValues = { crc32: 'AAAAAA==', crc32c: 'AQAAAA==', @@ -54,8 +51,7 @@ function makeListRequest() { }; } -function setOverviewChecksum(checksumAlgorithm, checksumType, - checksumIsDefault) { +function setOverviewChecksum(checksumAlgorithm, checksumType, checksumIsDefault) { const overview = inMemMetadata.keyMaps.get(mpuBucket).get(overviewKey); overview.checksumAlgorithm = checksumAlgorithm; overview.checksumType = checksumType; @@ -72,74 +68,82 @@ describe('List Parts API', () => { beforeEach(done => { cleanup(); const creationDate = new Date().toJSON(); - const sampleNormalBucketInstance = new BucketInfo(bucketName, - canonicalID, authInfo.getAccountDisplayName(), creationDate, - BucketInfo.currentModelVersion()); - const sampleMPUInstance = new BucketInfo(mpuBucket, - 'admin', 'admin', creationDate, BucketInfo.currentModelVersion()); - metadata.createBucket(bucketName, sampleNormalBucketInstance, log, - () => { - metadata.createBucket(mpuBucket, sampleMPUInstance, log, () => { - inMemMetadata.keyMaps.get(mpuBucket).set(overviewKey, { - 'id': '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', - 'owner-display-name': authInfo.getAccountDisplayName(), - 'owner-id': canonicalID, - 'initiator': { - DisplayName: authInfo.getAccountDisplayName(), - ID: canonicalID, - }, - 'key': '$makememulti', - 'initiated': '2015-11-30T22:40:07.858Z', - 'uploadId': '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - 'eventualStorageBucket': 'freshestbucket', - 'mdBucketModelVersion': 2, - }); - - inMemMetadata.keyMaps.get(mpuBucket).set(partOneKey, { - 'key': partOneKey, - 'last-modified': '2015-11-30T22:41:18.658Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['068db6a6745a79d54c1b29ff99f9f131'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partTwoKey, { - 'key': partTwoKey, - 'last-modified': '2015-11-30T22:41:40.207Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['ff22f316b16956ff5118c93abce7d62d'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partThreeKey, { - 'key': partThreeKey, - 'last-modified': '2015-11-30T22:41:52.102Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['dea282f70edb6fc5f9433cd6f525d4a6'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partFourKey, { - 'key': partFourKey, - 'last-modified': '2015-11-30T22:42:03.493Z', - 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', - 'content-length': '6000000', - 'partLocations': ['afe24bc40153982e1f7f28066f7af6a4'], - }); - inMemMetadata.keyMaps.get(mpuBucket).set(partFiveKey, { - 'key': partFiveKey, - 'last-modified': '2015-11-30T22:42:22.876Z', - 'content-md5': '555e4cd2f9eff38109d7a3ab13995a32', - 'content-length': '18', - 'partLocations': ['85bc16f5769687070fb13cfe66b5e41f'], - }); - done(); + const sampleNormalBucketInstance = new BucketInfo( + bucketName, + canonicalID, + authInfo.getAccountDisplayName(), + creationDate, + BucketInfo.currentModelVersion(), + ); + const sampleMPUInstance = new BucketInfo( + mpuBucket, + 'admin', + 'admin', + creationDate, + BucketInfo.currentModelVersion(), + ); + metadata.createBucket(bucketName, sampleNormalBucketInstance, log, () => { + metadata.createBucket(mpuBucket, sampleMPUInstance, log, () => { + inMemMetadata.keyMaps.get(mpuBucket).set(overviewKey, { + id: '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', + 'owner-display-name': authInfo.getAccountDisplayName(), + 'owner-id': canonicalID, + initiator: { + DisplayName: authInfo.getAccountDisplayName(), + ID: canonicalID, + }, + key: '$makememulti', + initiated: '2015-11-30T22:40:07.858Z', + uploadId: '4db92ccc-d89d-49d3-9fa6-e9c2c1eb31b0', + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + eventualStorageBucket: 'freshestbucket', + mdBucketModelVersion: 2, + }); + + inMemMetadata.keyMaps.get(mpuBucket).set(partOneKey, { + key: partOneKey, + 'last-modified': '2015-11-30T22:41:18.658Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['068db6a6745a79d54c1b29ff99f9f131'], }); + inMemMetadata.keyMaps.get(mpuBucket).set(partTwoKey, { + key: partTwoKey, + 'last-modified': '2015-11-30T22:41:40.207Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['ff22f316b16956ff5118c93abce7d62d'], + }); + inMemMetadata.keyMaps.get(mpuBucket).set(partThreeKey, { + key: partThreeKey, + 'last-modified': '2015-11-30T22:41:52.102Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['dea282f70edb6fc5f9433cd6f525d4a6'], + }); + inMemMetadata.keyMaps.get(mpuBucket).set(partFourKey, { + key: partFourKey, + 'last-modified': '2015-11-30T22:42:03.493Z', + 'content-md5': 'f3a9fb2071d3503b703938a74eb99846', + 'content-length': '6000000', + partLocations: ['afe24bc40153982e1f7f28066f7af6a4'], + }); + inMemMetadata.keyMaps.get(mpuBucket).set(partFiveKey, { + key: partFiveKey, + 'last-modified': '2015-11-30T22:42:22.876Z', + 'content-md5': '555e4cd2f9eff38109d7a3ab13995a32', + 'content-length': '18', + partLocations: ['85bc16f5769687070fb13cfe66b5e41f'], + }); + done(); }); + }); }); it('should list all parts of a multipart upload', done => { @@ -153,32 +157,20 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1000'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '1'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '6000000'); - assert.strictEqual(json.ListPartsResult.Part[4].PartNumber[0], - '5'); - assert.strictEqual(json.ListPartsResult.Part[4].ETag[0], - lastPieceETag); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '1'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '6000000'); + assert.strictEqual(json.ListPartsResult.Part[4].PartNumber[0], '5'); + assert.strictEqual(json.ListPartsResult.Part[4].ETag[0], lastPieceETag); assert.strictEqual(json.ListPartsResult.Part[4].Size[0], '18'); assert.strictEqual(json.ListPartsResult.Part.length, 5); - assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, - undefined); - assert.strictEqual(json.ListPartsResult.ChecksumType, - undefined); - assert.strictEqual(json.ListPartsResult.Part[0].ChecksumSHA256, - undefined); + assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, undefined); + assert.strictEqual(json.ListPartsResult.ChecksumType, undefined); + assert.strictEqual(json.ListPartsResult.Part[0].ChecksumSHA256, undefined); done(); }); }); @@ -194,15 +186,10 @@ describe('List Parts API', () => { assert.strictEqual(err, null); parseString(xml, (err, json) => { assert.strictEqual(err, null); - assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm[0], - 'SHA256'); - assert.strictEqual(json.ListPartsResult.ChecksumType[0], - 'COMPOSITE'); - assert.strictEqual( - json.ListPartsResult.Part[0].ChecksumSHA256[0], - partChecksumValues.sha256); - assert.strictEqual(json.ListPartsResult.Part[1].ChecksumSHA256, - undefined); + assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm[0], 'SHA256'); + assert.strictEqual(json.ListPartsResult.ChecksumType[0], 'COMPOSITE'); + assert.strictEqual(json.ListPartsResult.Part[0].ChecksumSHA256[0], partChecksumValues.sha256); + assert.strictEqual(json.ListPartsResult.Part[1].ChecksumSHA256, undefined); done(); }); }); @@ -210,8 +197,7 @@ describe('List Parts API', () => { it('should omit default CRC64NVME checksum fields', done => { setOverviewChecksum('crc64nvme', 'FULL_OBJECT', true); - setPartChecksum(partOneKey, 'crc64nvme', - partChecksumValues.crc64nvme); + setPartChecksum(partOneKey, 'crc64nvme', partChecksumValues.crc64nvme); const listRequest = makeListRequest(); @@ -219,20 +205,15 @@ describe('List Parts API', () => { assert.strictEqual(err, null); parseString(xml, (err, json) => { assert.strictEqual(err, null); - assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, - undefined); - assert.strictEqual(json.ListPartsResult.ChecksumType, - undefined); - assert.strictEqual( - json.ListPartsResult.Part[0].ChecksumCRC64NVME, - undefined); + assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, undefined); + assert.strictEqual(json.ListPartsResult.ChecksumType, undefined); + assert.strictEqual(json.ListPartsResult.Part[0].ChecksumCRC64NVME, undefined); done(); }); }); }); - it('should omit root checksum fields if only checksumAlgorithm is set', - done => { + it('should omit root checksum fields if only checksumAlgorithm is set', done => { setOverviewChecksum('sha256', undefined, false); const listRequest = makeListRequest(); @@ -241,10 +222,8 @@ describe('List Parts API', () => { assert.strictEqual(err, null); parseString(xml, (err, json) => { assert.strictEqual(err, null); - assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, - undefined); - assert.strictEqual(json.ListPartsResult.ChecksumType, - undefined); + assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, undefined); + assert.strictEqual(json.ListPartsResult.ChecksumType, undefined); done(); }); }); @@ -259,17 +238,14 @@ describe('List Parts API', () => { assert.strictEqual(err, null); parseString(xml, (err, json) => { assert.strictEqual(err, null); - assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, - undefined); - assert.strictEqual(json.ListPartsResult.ChecksumType, - undefined); + assert.strictEqual(json.ListPartsResult.ChecksumAlgorithm, undefined); + assert.strictEqual(json.ListPartsResult.ChecksumType, undefined); done(); }); }); }); - it('should render checksum tags for every supported part algorithm', - done => { + it('should render checksum tags for every supported part algorithm', done => { setOverviewChecksum('sha256', 'COMPOSITE', false); const checksumCases = [ @@ -280,8 +256,7 @@ describe('List Parts API', () => { ['sha256', 'ChecksumSHA256'], ]; checksumCases.forEach(([algorithm], index) => { - setPartChecksum(partKeys[index], algorithm, - partChecksumValues[algorithm]); + setPartChecksum(partKeys[index], algorithm, partChecksumValues[algorithm]); }); const listRequest = makeListRequest(); @@ -291,9 +266,7 @@ describe('List Parts API', () => { parseString(xml, (err, json) => { assert.strictEqual(err, null); checksumCases.forEach(([algorithm, xmlTag], index) => { - assert.strictEqual( - json.ListPartsResult.Part[index][xmlTag][0], - partChecksumValues[algorithm]); + assert.strictEqual(json.ListPartsResult.Part[index][xmlTag][0], partChecksumValues[algorithm]); }); done(); }); @@ -307,16 +280,18 @@ describe('List Parts API', () => { data.listParts = (mpuInfo, request, lcCheckFn, log, callback) => callback(null, { IsTruncated: false, - Contents: [{ - partNumber: 1, - value: { - LastModified: '2015-11-30T22:41:18.658Z', - ETag: 'f3a9fb2071d3503b703938a74eb99846', - Size: 6000000, - ChecksumAlgorithm: 'sha256', - ChecksumValue: partChecksumValues.sha256, + Contents: [ + { + partNumber: 1, + value: { + LastModified: '2015-11-30T22:41:18.658Z', + ETag: 'f3a9fb2071d3503b703938a74eb99846', + Size: 6000000, + ChecksumAlgorithm: 'sha256', + ChecksumValue: partChecksumValues.sha256, + }, }, - }], + ], }); const listRequest = makeListRequest(); @@ -327,11 +302,8 @@ describe('List Parts API', () => { parseString(xml, (err, json) => { assert.strictEqual(err, null); assert.strictEqual(json.ListPartsResult.Part.length, 1); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '1'); - assert.strictEqual( - json.ListPartsResult.Part[0].ChecksumSHA256[0], - partChecksumValues.sha256); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '1'); + assert.strictEqual(json.ListPartsResult.Part[0].ChecksumSHA256[0], partChecksumValues.sha256); done(); }); }); @@ -355,15 +327,13 @@ describe('List Parts API', () => { listParts(authInfo, listRequest, log, (err, xml) => { assert.strictEqual(err, null); parseString(xml, (err, json) => { - assert.strictEqual(json.ListPartsResult.Key[0], - urlEncodedObjectKey); + assert.strictEqual(json.ListPartsResult.Key[0], urlEncodedObjectKey); done(); }); }); }); - it('should list only up to requested number ' + - 'of max parts of a multipart upload', done => { + it('should list only up to requested number ' + 'of max parts of a multipart upload', done => { const listRequest = { bucketName, namespace, @@ -385,27 +355,20 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '4'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'true'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], - '4'); - assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[2].Size[0], - '6000000'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], '4'); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[2].Size[0], '6000000'); assert.strictEqual(json.ListPartsResult.Part.length, 4); done(); }); }); }); - it('should list all parts if requested max-parts ' + - 'is greater than total number of parts', done => { + it('should list all parts if requested max-parts ' + 'is greater than total number of parts', done => { const listRequest = { bucketName, namespace, @@ -427,20 +390,13 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '6'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[2].Size[0], - '6000000'); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[2].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[2].Size[0], '6000000'); assert.strictEqual(json.ListPartsResult.Part.length, 5); done(); }); @@ -469,30 +425,21 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '1000'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); - assert.strictEqual(json.ListPartsResult.IsTruncated[0], - 'false'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], - '2'); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, - undefined); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '6000000'); - assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], - '5'); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'false'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], '2'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker, undefined); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '6000000'); + assert.strictEqual(json.ListPartsResult.Part[2].PartNumber[0], '5'); assert.strictEqual(json.ListPartsResult.Part.length, 3); done(); }); }); }); - it('should handle a part-number-marker specified ' + - 'and a max-parts specified', done => { + it('should handle a part-number-marker specified ' + 'and a max-parts specified', done => { const listRequest = { bucketName, namespace, @@ -515,21 +462,14 @@ describe('List Parts API', () => { assert.strictEqual(json.ListPartsResult.Key[0], uploadKey); assert.strictEqual(json.ListPartsResult.UploadId[0], uploadId); assert.strictEqual(json.ListPartsResult.MaxParts[0], '2'); - assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], - authInfo.getCanonicalID()); + assert.strictEqual(json.ListPartsResult.Initiator[0].ID[0], authInfo.getCanonicalID()); assert.strictEqual(json.ListPartsResult.IsTruncated[0], 'true'); - assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], - '2'); - assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], - '4'); - assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], - '3'); - assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], - sixMBObjectETag); - assert.strictEqual(json.ListPartsResult.Part[0].Size[0], - '6000000'); - assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], - '4'); + assert.strictEqual(json.ListPartsResult.PartNumberMarker[0], '2'); + assert.strictEqual(json.ListPartsResult.NextPartNumberMarker[0], '4'); + assert.strictEqual(json.ListPartsResult.Part[0].PartNumber[0], '3'); + assert.strictEqual(json.ListPartsResult.Part[0].ETag[0], sixMBObjectETag); + assert.strictEqual(json.ListPartsResult.Part[0].Size[0], '6000000'); + assert.strictEqual(json.ListPartsResult.Part[1].PartNumber[0], '4'); assert.strictEqual(json.ListPartsResult.Part.length, 2); done(); }); diff --git a/tests/unit/api/multipartDelete.js b/tests/unit/api/multipartDelete.js index be9e9ec81e..dae41d6491 100644 --- a/tests/unit/api/multipartDelete.js +++ b/tests/unit/api/multipartDelete.js @@ -6,8 +6,7 @@ const { cleanup, DummyRequestLogger } = require('../helpers'); const { config } = require('../../../lib/Config'); const DummyRequest = require('../DummyRequest'); const { bucketPut } = require('../../../lib/api/bucketPut'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const multipartDelete = require('../../../lib/api/multipartDelete'); const objectPutPart = require('../../../lib/api/objectPutPart'); const { makeAuthInfo } = require('../helpers'); @@ -36,62 +35,63 @@ const initiateRequest = { const eastLocation = 'us-east-1'; const westLocation = 'scality-internal-file'; -function _createAndAbortMpu(usEastSetting, fakeUploadID, locationConstraint, - callback) { - config.locationConstraints['us-east-1'].legacyAwsBehavior = - usEastSetting; - const post = '' + +function _createAndAbortMpu(usEastSetting, fakeUploadID, locationConstraint, callback) { + config.locationConstraints['us-east-1'].legacyAwsBehavior = usEastSetting; + const post = + '' + '' + `${locationConstraint}` + ''; const testBucketPutRequest = Object.assign({ post }, bucketPutRequest); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => - initiateMultipartUpload(authInfo, initiateRequest, log, next), - (result, corsHeaders, next) => parseString(result, next), - (json, next) => { - // use uploadId parsed from initiateMpu request to construct - // uploadPart and deleteMpu requests - const uploadId = - json.InitiateMultipartUploadResult.UploadId[0]; - const partBody = Buffer.from('I am a part\n', 'utf8'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, - query: { - partNumber: '1', - uploadId, - }, - actionImplicitDenies: false, - }, partBody); - const testUploadId = fakeUploadID ? 'nonexistinguploadid' : - uploadId; - const deleteMpuRequest = { - bucketName, - namespace, - objectKey, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectKey}?uploadId=${testUploadId}`, - query: { uploadId: testUploadId }, - actionImplicitDenies: false, - }; - next(null, partRequest, deleteMpuRequest); - }, - (partRequest, deleteMpuRequest, next) => - objectPutPart(authInfo, partRequest, undefined, log, err => { - if (err) { - return next(err); - } - return next(null, deleteMpuRequest); - }), - (deleteMpuRequest, next) => - multipartDelete(authInfo, deleteMpuRequest, log, next), - ], callback); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), + (result, corsHeaders, next) => parseString(result, next), + (json, next) => { + // use uploadId parsed from initiateMpu request to construct + // uploadPart and deleteMpu requests + const uploadId = json.InitiateMultipartUploadResult.UploadId[0]; + const partBody = Buffer.from('I am a part\n', 'utf8'); + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?partNumber=1&uploadId=${uploadId}`, + query: { + partNumber: '1', + uploadId, + }, + actionImplicitDenies: false, + }, + partBody, + ); + const testUploadId = fakeUploadID ? 'nonexistinguploadid' : uploadId; + const deleteMpuRequest = { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectKey}?uploadId=${testUploadId}`, + query: { uploadId: testUploadId }, + actionImplicitDenies: false, + }; + next(null, partRequest, deleteMpuRequest); + }, + (partRequest, deleteMpuRequest, next) => + objectPutPart(authInfo, partRequest, undefined, log, err => { + if (err) { + return next(err); + } + return next(null, deleteMpuRequest); + }), + (deleteMpuRequest, next) => multipartDelete(authInfo, deleteMpuRequest, log, next), + ], + callback, + ); } describe('Multipart Delete API', () => { @@ -100,41 +100,47 @@ describe('Multipart Delete API', () => { }); afterEach(() => { // set back to original - config.locationConstraints['us-east-1'].legacyAwsBehavior = - true; + config.locationConstraints['us-east-1'].legacyAwsBehavior = true; cleanup(); }); - it('should not return error if mpu exists with uploadId and at least ' + - 'one part', done => { + it('should not return error if mpu exists with uploadId and at least ' + 'one part', done => { _createAndAbortMpu(true, false, eastLocation, err => { assert.ifError(err); done(err); }); }); - it('should still not return error if uploadId does not exist on ' + - 'multipart abort call, in region other than us-east-1', done => { - _createAndAbortMpu(true, true, westLocation, err => { - assert.ifError(err); - done(err); - }); - }); + it( + 'should still not return error if uploadId does not exist on ' + + 'multipart abort call, in region other than us-east-1', + done => { + _createAndAbortMpu(true, true, westLocation, err => { + assert.ifError(err); + done(err); + }); + }, + ); - it('bucket created in us-east-1: should return 404 if uploadId does not ' + - 'exist and legacyAwsBehavior set to true', - done => { - _createAndAbortMpu(true, true, eastLocation, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); - }); + it( + 'bucket created in us-east-1: should return 404 if uploadId does not ' + + 'exist and legacyAwsBehavior set to true', + done => { + _createAndAbortMpu(true, true, eastLocation, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); + }, + ); - it('bucket created in us-east-1: should return no error ' + - 'if uploadId does not exist and legacyAwsBehavior set to false', done => { - _createAndAbortMpu(false, true, eastLocation, err => { - assert.strictEqual(err, null, `Expected no error, got ${err}`); - done(); - }); - }); + it( + 'bucket created in us-east-1: should return no error ' + + 'if uploadId does not exist and legacyAwsBehavior set to false', + done => { + _createAndAbortMpu(false, true, eastLocation, err => { + assert.strictEqual(err, null, `Expected no error, got ${err}`); + done(); + }); + }, + ); }); diff --git a/tests/unit/api/objectACLauth.js b/tests/unit/api/objectACLauth.js index df0ffad322..3edc896581 100644 --- a/tests/unit/api/objectACLauth.js +++ b/tests/unit/api/objectACLauth.js @@ -2,8 +2,7 @@ const assert = require('assert'); const BucketInfo = require('arsenal').models.BucketInfo; const constants = require('../../../constants'); -const { isObjAuthorized } - = require('../../../lib/api/apiUtils/authorization/permissionChecks'); +const { isObjAuthorized } = require('../../../lib/api/apiUtils/authorization/permissionChecks'); const { DummyRequestLogger, makeAuthInfo } = require('../helpers'); const accessKey = 'accessKey1'; @@ -15,12 +14,11 @@ const userAuthInfo = makeAuthInfo(accessKey, 'user'); const altAcctAuthInfo = makeAuthInfo(altAccessKey); const accountToVet = altAcctAuthInfo.getCanonicalID(); -const bucket = new BucketInfo('niftyBucket', bucketOwnerCanonicalId, - 'iAmTheOwnerDisplayName', creationDate); +const bucket = new BucketInfo('niftyBucket', bucketOwnerCanonicalId, 'iAmTheOwnerDisplayName', creationDate); const objectOwnerCanonicalId = userAuthInfo.getCanonicalID(); const object = { 'owner-id': objectOwnerCanonicalId, - 'acl': { + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], @@ -46,60 +44,61 @@ describe('object acl authorization for objectGet and objectHead', () => { it('should allow access to object owner', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - authInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to user in object owner account', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - userAuthInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, userAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to bucket owner if same account as object owner', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, bucketOwnerCanonicalId, - authInfo, log)); + isObjAuthorized(bucket, object, type, bucketOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to anyone if canned public-read ACL', () => { object.acl.Canned = 'public-read'; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to anyone if canned public-read-write ACL', () => { object.acl.Canned = 'public-read-write'; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); - it('should not allow access to public user if ' + - 'authenticated-read ACL', () => { + it('should not allow access to public user if ' + 'authenticated-read ACL', () => { object.acl.Canned = 'authenticated-read'; const publicResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, constants.publicId, null, log)); + isObjAuthorized(bucket, object, type, constants.publicId, null, log), + ); assert.deepStrictEqual(publicResults, [false, false]); }); - it('should allow access to any authenticated user if ' + - 'authenticated-read ACL', () => { + it('should allow access to any authenticated user if ' + 'authenticated-read ACL', () => { object.acl.Canned = 'authenticated-read'; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); - it('should allow access to bucket owner when object owner is alt account if ' + - 'bucket-owner-read ACL', () => { + it('should allow access to bucket owner when object owner is alt account if ' + 'bucket-owner-read ACL', () => { const altAcctObj = { 'owner-id': accountToVet, - 'acl': { + acl: { Canned: 'private', FULL_CONTROL: [], WRITE_ACP: [], @@ -108,70 +107,76 @@ describe('object acl authorization for objectGet and objectHead', () => { }, }; const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); altAcctObj.acl.Canned = 'bucket-owner-read'; const authResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(authResults, [true, true]); }); - it('should allow access to bucket owner when object owner is alt account if ' + - 'bucket-owner-full-control ACL', () => { - const altAcctObj = { - 'owner-id': accountToVet, - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - }; - const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); - assert.deepStrictEqual(noAuthResults, [false, false]); - altAcctObj.acl.Canned = 'bucket-owner-full-control'; - const authResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); - assert.deepStrictEqual(authResults, [true, true]); - }); + it( + 'should allow access to bucket owner when object owner is alt account if ' + 'bucket-owner-full-control ACL', + () => { + const altAcctObj = { + 'owner-id': accountToVet, + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + }; + const noAuthResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); + assert.deepStrictEqual(noAuthResults, [false, false]); + altAcctObj.acl.Canned = 'bucket-owner-full-control'; + const authResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); + assert.deepStrictEqual(authResults, [true, true]); + }, + ); - it('should allow access to account if ' + - 'account was granted FULL_CONTROL', () => { + it('should allow access to account if ' + 'account was granted FULL_CONTROL', () => { const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); object.acl.FULL_CONTROL = [accountToVet]; const authResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(authResults, [true, true]); }); - it('should allow access to account if ' + - 'account was granted READ right', () => { + it('should allow access to account if ' + 'account was granted READ right', () => { const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); object.acl.READ = [accountToVet]; const authResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(authResults, [true, true]); }); it('should not allow access to public user if private canned ACL', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [false, false]); }); it('should not allow access to just any user if private canned ACL', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [false, false]); }); }); @@ -180,11 +185,12 @@ describe('object authorization for objectPut and objectDelete', () => { it('should allow access when no implicitDeny information is provided', () => { const requestTypes = ['objectPut', 'objectDelete']; const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); const publicUserResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, constants.publicId, null, - log)); + isObjAuthorized(bucket, object, type, constants.publicId, null, log), + ); assert.deepStrictEqual(publicUserResults, [true, true]); }); }); @@ -206,71 +212,69 @@ describe('object authorization for objectPutACL and objectGetACL', () => { it('should allow access to object owner', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - authInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, authInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); it('should allow access to user in object owner account', () => { const results = requestTypes.map(type => - isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, - userAuthInfo, log)); + isObjAuthorized(bucket, object, type, objectOwnerCanonicalId, userAuthInfo, log), + ); assert.deepStrictEqual(results, [true, true]); }); - it('should allow access to bucket owner when object owner is alt account if ' + - 'bucket-owner-full-control canned ACL set on object', () => { - const altAcctObj = { - 'owner-id': accountToVet, - 'acl': { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }, - }; - const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - log)); - assert.deepStrictEqual(noAuthResults, [false, false]); - altAcctObj.acl.Canned = 'bucket-owner-full-control'; - const authorizedResults = requestTypes.map(type => - isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, - null, log)); - assert.deepStrictEqual(authorizedResults, [true, true]); - }); + it( + 'should allow access to bucket owner when object owner is alt account if ' + + 'bucket-owner-full-control canned ACL set on object', + () => { + const altAcctObj = { + 'owner-id': accountToVet, + acl: { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + }; + const noAuthResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, log), + ); + assert.deepStrictEqual(noAuthResults, [false, false]); + altAcctObj.acl.Canned = 'bucket-owner-full-control'; + const authorizedResults = requestTypes.map(type => + isObjAuthorized(bucket, altAcctObj, type, bucketOwnerCanonicalId, authInfo, null, log), + ); + assert.deepStrictEqual(authorizedResults, [true, true]); + }, + ); - it('should allow access to account if ' + - 'account was granted FULL_CONTROL right', () => { + it('should allow access to account if ' + 'account was granted FULL_CONTROL right', () => { const noAuthResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(noAuthResults, [false, false]); object.acl.FULL_CONTROL = [accountToVet]; const authorizedResults = requestTypes.map(type => - isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log)); + isObjAuthorized(bucket, object, type, accountToVet, altAcctAuthInfo, log), + ); assert.deepStrictEqual(authorizedResults, [true, true]); }); - it('should allow objectPutACL access to account if ' + - 'account was granted WRITE_ACP right', () => { - const noAuthResult = isObjAuthorized(bucket, object, 'objectPutACL', - accountToVet, altAcctAuthInfo, log); + it('should allow objectPutACL access to account if ' + 'account was granted WRITE_ACP right', () => { + const noAuthResult = isObjAuthorized(bucket, object, 'objectPutACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(noAuthResult, false); object.acl.WRITE_ACP = [accountToVet]; - const authorizedResult = isObjAuthorized(bucket, object, 'objectPutACL', - accountToVet, altAcctAuthInfo, log); + const authorizedResult = isObjAuthorized(bucket, object, 'objectPutACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(authorizedResult, true); }); - it('should allow objectGetACL access to account if ' + - 'account was granted READ_ACP right', () => { - const noAuthResult = isObjAuthorized(bucket, object, 'objectGetACL', - accountToVet, altAcctAuthInfo, log); + it('should allow objectGetACL access to account if ' + 'account was granted READ_ACP right', () => { + const noAuthResult = isObjAuthorized(bucket, object, 'objectGetACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(noAuthResult, false); object.acl.READ_ACP = [accountToVet]; - const authorizedResult = isObjAuthorized(bucket, object, 'objectGetACL', - accountToVet, altAcctAuthInfo, log); + const authorizedResult = isObjAuthorized(bucket, object, 'objectGetACL', accountToVet, altAcctAuthInfo, log); assert.strictEqual(authorizedResult, true); }); }); @@ -288,12 +292,7 @@ describe('without object metadata', () => { bucket.setBucketPolicy(null); }); - const requestTypes = [ - 'objectGet', - 'objectHead', - 'objectPutACL', - 'objectGetACL', - ]; + const requestTypes = ['objectGet', 'objectHead', 'objectPutACL', 'objectGetACL']; const allowedAccess = [true, true, true, true]; const deniedAccess = [false, false, false, false]; @@ -301,77 +300,87 @@ describe('without object metadata', () => { const tests = [ { it: 'should allow user if part of the bucket owner account', - canned: 'private', id: objectOwnerCanonicalId, + canned: 'private', + id: objectOwnerCanonicalId, authInfo: userAuthInfo, aclParam: null, response: allowedAccess, }, { it: 'should not allow user if not part of the bucket owner account', - canned: 'private', id: accountToVet, + canned: 'private', + id: accountToVet, authInfo: altAcctAuthInfo, aclParam: null, response: deniedAccess, }, { it: 'should allow bucket owner', - canned: 'private', id: bucketOwnerCanonicalId, + canned: 'private', + id: bucketOwnerCanonicalId, aclParam: null, response: allowedAccess, }, { it: 'should not allow public if canned private', - canned: 'private', id: constants.publicId, + canned: 'private', + id: constants.publicId, aclParam: null, response: deniedAccess, }, { it: 'should not allow other accounts if canned private', - canned: 'private', id: accountToVet, + canned: 'private', + id: accountToVet, aclParam: null, response: deniedAccess, }, { it: 'should allow public if bucket is canned public-read', - canned: 'public-read', id: constants.publicId, + canned: 'public-read', + id: constants.publicId, aclParam: null, response: allowedAccess, }, { it: 'should allow public if bucket is canned public-read-write', - canned: 'public-read-write', id: constants.publicId, + canned: 'public-read-write', + id: constants.publicId, aclParam: null, response: allowedAccess, }, { - it: 'should not allow public if bucket is canned ' + - 'authenticated-read', - canned: 'authenticated-read', id: constants.publicId, + it: 'should not allow public if bucket is canned ' + 'authenticated-read', + canned: 'authenticated-read', + id: constants.publicId, aclParam: null, response: deniedAccess, }, { - it: 'should allow authenticated users if bucket is canned ' + - 'authenticated-read', - canned: 'authenticated-read', id: accountToVet, + it: 'should allow authenticated users if bucket is canned ' + 'authenticated-read', + canned: 'authenticated-read', + id: accountToVet, aclParam: null, response: allowedAccess, }, { it: 'should allow account if granted bucket READ', - canned: '', id: accountToVet, + canned: '', + id: accountToVet, aclParam: ['READ', accountToVet], response: allowedAccess, }, { it: 'should allow account if granted bucket FULL_CONTROL', - canned: '', id: accountToVet, + canned: '', + id: accountToVet, aclParam: ['FULL_CONTROL', accountToVet], response: allowedAccess, }, { it: 'should allow public if granted bucket read action in policy', - canned: 'private', id: constants.publicId, + canned: 'private', + id: constants.publicId, aclParam: null, policy: { Version: '2012-10-17', @@ -388,7 +397,8 @@ describe('without object metadata', () => { }, { it: 'should not allow public if denied bucket read action in policy', - canned: 'public-read', id: constants.publicId, + canned: 'public-read', + id: constants.publicId, aclParam: null, policy: { Version: '2012-10-17', @@ -405,7 +415,8 @@ describe('without object metadata', () => { }, { it: 'should allow account if granted bucket read action in policy', - canned: 'private', id: accountToVet, + canned: 'private', + id: accountToVet, aclParam: null, policy: { Version: '2012-10-17', @@ -423,7 +434,8 @@ describe('without object metadata', () => { }, { it: 'should not allow account if denied bucket read action in policy', - canned: 'public-read', id: accountToVet, + canned: 'public-read', + id: accountToVet, aclParam: null, policy: { Version: '2012-10-17', @@ -454,8 +466,7 @@ describe('without object metadata', () => { } bucket.setCannedAcl(value.canned); - const results = requestTypes.map(type => - isObjAuthorized(bucket, null, type, value.id, authInfoUser, log)); + const results = requestTypes.map(type => isObjAuthorized(bucket, null, type, value.id, authInfoUser, log)); assert.deepStrictEqual(results, value.response); done(); }); @@ -473,8 +484,7 @@ describe('without object metadata', () => { }, ], }); - const results = isObjAuthorized(bucket, null, 'initiateMultipartUpload', - accountToVet, altAcctAuthInfo, log); + const results = isObjAuthorized(bucket, null, 'initiateMultipartUpload', accountToVet, altAcctAuthInfo, log); assert.strictEqual(results, true); }); @@ -490,8 +500,7 @@ describe('without object metadata', () => { }, ], }); - const results = isObjAuthorized(bucket, null, 'objectPutPart', - accountToVet, altAcctAuthInfo, log); + const results = isObjAuthorized(bucket, null, 'objectPutPart', accountToVet, altAcctAuthInfo, log); assert.strictEqual(results, true); }); @@ -507,8 +516,7 @@ describe('without object metadata', () => { }, ], }); - const results = isObjAuthorized(bucket, null, 'completeMultipartUpload', - accountToVet, altAcctAuthInfo, log); + const results = isObjAuthorized(bucket, null, 'completeMultipartUpload', accountToVet, altAcctAuthInfo, log); assert.strictEqual(results, true); }); }); diff --git a/tests/unit/api/objectDelete.js b/tests/unit/api/objectDelete.js index 2e02455602..5acf3d8fd3 100644 --- a/tests/unit/api/objectDelete.js +++ b/tests/unit/api/objectDelete.js @@ -7,7 +7,7 @@ const services = require('../../../lib/services'); const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutACL = require('../../../lib/api/bucketPutACL'); const constants = require('../../../constants'); -const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils} = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, versioningTestUtils } = require('../helpers'); const objectPut = require('../../../lib/api/objectPut'); const { objectDelete, objectDeleteInternal } = require('../../../lib/api/objectDelete'); const objectGet = require('../../../lib/api/objectGet'); @@ -35,8 +35,7 @@ lateDate.setMinutes(lateDate.getMinutes() + 30); const enableVersioningRequest = versioningTestUtils.createBucketPutVersioningReq(bucketName, 'Enabled'); -function testAuth(bucketOwner, authUser, bucketPutReq, objPutReq, objDelReq, - log, cb) { +function testAuth(bucketOwner, authUser, bucketPutReq, objPutReq, objDelReq, log, cb) { bucketPut(bucketOwner, bucketPutReq, log, () => { bucketPutACL(bucketOwner, bucketPutReq, log, err => { assert.strictEqual(err, undefined); @@ -56,13 +55,16 @@ describe('objectDelete API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + postBody, + ); sinon.stub(services, 'deleteObject').callsFake(originalDeleteObject); sinon.spy(metadataswitch, 'putObjectMD'); @@ -73,7 +75,6 @@ describe('objectDelete API', () => { sinon.restore(); }); - const testBucketPutRequest = new DummyRequest({ bucketName, namespace, @@ -96,43 +97,65 @@ describe('objectDelete API', () => { }); it('should delete an object', done => { - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - doesNotNeedOpogUpdate: true, - }), - any, any, any), - next => objectGet(authInfo, testGetObjectRequest, false, log, err => { - assert.strictEqual(err.is.NoSuchKey, true); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + doesNotNeedOpogUpdate: true, + }), + any, + any, + any, + ), + next => + objectGet(authInfo, testGetObjectRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchKey, true); + next(); + }), + ], + done, + ); }); it('should delete an object with oplog update when object is archived', done => { const archived = { archiveInfo: { foo: 0, bar: 'stuff' } }; - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - doesNotNeedOpogUpdate: undefined, - }), - any, any, any), - next => objectGet(authInfo, testGetObjectRequest, false, log, err => { - assert.strictEqual(err.is.NoSuchKey, true); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => fakeMetadataArchive(bucketName, objectKey, undefined, archived, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + doesNotNeedOpogUpdate: undefined, + }), + any, + any, + any, + ), + next => + objectGet(authInfo, testGetObjectRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchKey, true); + next(); + }), + ], + done, + ); }); it('should delete an object with oplog update when bucket has bucket notification', done => { @@ -141,72 +164,90 @@ describe('objectDelete API', () => { headers: { host: `${bucketName}.s3.amazonaws.com`, }, - post: '' + + post: + '' + '', actionImplicitDenies: false, }; - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => bucketPutNotification(authInfo, putNotifConfigRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - doesNotNeedOpogUpdate: undefined, - }), - any, any, any), - next => objectGet(authInfo, testGetObjectRequest, false, log, err => { - assert.strictEqual(err.is.NoSuchKey, true); - next(); - }), - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => bucketPutNotification(authInfo, putNotifConfigRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + doesNotNeedOpogUpdate: undefined, + }), + any, + any, + any, + ), + next => + objectGet(authInfo, testGetObjectRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchKey, true); + next(); + }), + ], + done, + ); }); it('should delete a 0 bytes object', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, ''); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + '', + ); bucketPut(authInfo, testBucketPutRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, () => { - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err, null); - objectGet(authInfo, testGetObjectRequest, false, - log, err => { - const expected = - Object.assign({}, errors.NoSuchKey); - const received = Object.assign({}, err); - assert.deepStrictEqual(received, expected); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err, null); + objectGet(authInfo, testGetObjectRequest, false, log, err => { + const expected = Object.assign({}, errors.NoSuchKey); + const received = Object.assign({}, err); + assert.deepStrictEqual(received, expected); + done(); }); }); + }); }); }); it('should delete a multipart upload and send `uploadId` as `replayId` to deleteObject', done => { bucketPut(authInfo, testBucketPutRequest, log, () => { - mpuUtils.createMPU(namespace, bucketName, objectKey, log, - (err, testUploadId) => { - assert.ifError(err); - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err, null); - sinon.assert.calledWith(services.deleteObject, - any, any, any, - sinon.match({ - deleteData: true, - replayId: testUploadId, - doesNotNeedOpogUpdate: true, - }), any, any, any); - done(); - }); + mpuUtils.createMPU(namespace, bucketName, objectKey, log, (err, testUploadId) => { + assert.ifError(err); + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err, null); + sinon.assert.calledWith( + services.deleteObject, + any, + any, + any, + sinon.match({ + deleteData: true, + replayId: testUploadId, + doesNotNeedOpogUpdate: true, + }), + any, + any, + any, + ); + done(); }); + }); }); }); @@ -223,46 +264,40 @@ describe('objectDelete API', () => { it('should del object if user has FULL_CONTROL grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testBucketPutRequest.headers['x-amz-grant-full-control'] = - `id=${authUser.getCanonicalID()}`; - testAuth(bucketOwner, authUser, testBucketPutRequest, - testPutObjectRequest, testDeleteRequest, log, done); + testBucketPutRequest.headers['x-amz-grant-full-control'] = `id=${authUser.getCanonicalID()}`; + testAuth(bucketOwner, authUser, testBucketPutRequest, testPutObjectRequest, testDeleteRequest, log, done); }); it('should del object if user has WRITE grant on bucket', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); - testBucketPutRequest.headers['x-amz-grant-write'] = - `id=${authUser.getCanonicalID()}`; - testAuth(bucketOwner, authUser, testBucketPutRequest, - testPutObjectRequest, testDeleteRequest, log, done); + testBucketPutRequest.headers['x-amz-grant-write'] = `id=${authUser.getCanonicalID()}`; + testAuth(bucketOwner, authUser, testBucketPutRequest, testPutObjectRequest, testDeleteRequest, log, done); }); it('should del object in bucket with public-read-write acl', done => { const bucketOwner = makeAuthInfo('accessKey2'); const authUser = makeAuthInfo('accessKey3'); testBucketPutRequest.headers['x-amz-acl'] = 'public-read-write'; - testAuth(bucketOwner, authUser, testBucketPutRequest, - testPutObjectRequest, testDeleteRequest, log, done); + testAuth(bucketOwner, authUser, testBucketPutRequest, testPutObjectRequest, testDeleteRequest, log, done); }); it('should pass overheadField to metadata', done => { bucketPut(authInfo, testBucketPutRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, () => { - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err, null); - sinon.assert.calledWith( - metadataswitch.deleteObjectMD, - bucketName, - objectKey, - sinon.match({ overheadField: sinon.match.array }), - sinon.match.any, - sinon.match.any - ); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err, null); + sinon.assert.calledWith( + metadataswitch.deleteObjectMD, + bucketName, + objectKey, + sinon.match({ overheadField: sinon.match.array }), + sinon.match.any, + sinon.match.any, + ); + done(); }); + }); }); }); @@ -276,52 +311,54 @@ describe('objectDelete API', () => { }); bucketPut(authInfo, testBucketPutVersionRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, - undefined, log, (err, data) => { - const deleteObjectVersionRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}?versionId=${data['x-amz-version-id']}`, - query: { - versionId: data['x-amz-version-id'], - }, - }); - objectDeleteInternal(authInfo, deleteObjectVersionRequest, log, true, err => { - assert.strictEqual(err, null); - sinon.assert.calledWith(warnStub, 'expiration is trying to delete a master version ' + - 'of an object with versioning enabled'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, data) => { + const deleteObjectVersionRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}?versionId=${data['x-amz-version-id']}`, + query: { + versionId: data['x-amz-version-id'], + }, }); + objectDeleteInternal(authInfo, deleteObjectVersionRequest, log, true, err => { + assert.strictEqual(err, null); + sinon.assert.calledWith( + warnStub, + 'expiration is trying to delete a master version ' + 'of an object with versioning enabled', + ); + done(); + }); + }); }); }); - describe('with \'modified\' headers', () => { + describe("with 'modified' headers", () => { beforeEach(done => { bucketPut(authInfo, testBucketPutRequest, log, () => { objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); - it('should return error if request includes \'if-unmodified-since\' ' + - 'header and object has been modified', done => { - const testDeleteRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { 'if-unmodified-since': earlyDate }, - url: `/${bucketName}/${objectKey}`, - }); - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err.is.PreconditionFailed, true); - done(); - }); - }); + it( + "should return error if request includes 'if-unmodified-since' " + 'header and object has been modified', + done => { + const testDeleteRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { 'if-unmodified-since': earlyDate }, + url: `/${bucketName}/${objectKey}`, + }); + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err.is.PreconditionFailed, true); + done(); + }); + }, + ); - it('should delete an object with \'if-unmodified-since\' header', - done => { + it("should delete an object with 'if-unmodified-since' header", done => { const testDeleteRequest = new DummyRequest({ bucketName, namespace, @@ -331,31 +368,31 @@ describe('objectDelete API', () => { }); objectDelete(authInfo, testDeleteRequest, log, err => { assert.strictEqual(err, null); - objectGet(authInfo, testGetObjectRequest, false, log, - err => { + objectGet(authInfo, testGetObjectRequest, false, log, err => { assert.strictEqual(err.is.NoSuchKey, true); done(); }); }); }); - it('should return error if request includes \'if-modified-since\' ' + - 'header and object has not been modified', done => { - const testDeleteRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { 'if-modified-since': lateDate }, - url: `/${bucketName}/${objectKey}`, - }); - objectDelete(authInfo, testDeleteRequest, log, err => { - assert.strictEqual(err.is.NotModified, true); - done(); - }); - }); + it( + "should return error if request includes 'if-modified-since' " + 'header and object has not been modified', + done => { + const testDeleteRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { 'if-modified-since': lateDate }, + url: `/${bucketName}/${objectKey}`, + }); + objectDelete(authInfo, testDeleteRequest, log, err => { + assert.strictEqual(err.is.NotModified, true); + done(); + }); + }, + ); - it('should delete an object with \'if-modified-since\' header', - done => { + it("should delete an object with 'if-modified-since' header", done => { const testDeleteRequest = new DummyRequest({ bucketName, namespace, @@ -365,8 +402,7 @@ describe('objectDelete API', () => { }); objectDelete(authInfo, testDeleteRequest, log, err => { assert.strictEqual(err, null); - objectGet(authInfo, testGetObjectRequest, false, log, - err => { + objectGet(authInfo, testGetObjectRequest, false, log, err => { assert.strictEqual(err.is.NoSuchKey, true); done(); }); @@ -380,13 +416,16 @@ describe('objectDelete API with versioning', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: {}, - url: `/${bucketName}/${objectKey}`, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: {}, + url: `/${bucketName}/${objectKey}`, + }, + postBody, + ); sinon.stub(services, 'deleteObject').callsFake(originalDeleteObject); sinon.spy(metadataswitch, 'putObjectMD'); @@ -412,27 +451,44 @@ describe('objectDelete API with versioning', () => { }); it('should upgrade master-only document to a version document when storing a delete marker version', done => { - async.series([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), - next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), - next => objectDelete(authInfo, testDeleteRequest, log, next), - async () => { - const calls = metadataswitch.putObjectMD.getCalls(); - sinon.assert.calledWith(calls[calls.length - 2], - bucketName, objectKey, sinon.match({ - versionId: sinon.match.truthy, - isNull: true, - originOp: 's3:StoreNullVersion', - }), any, any, any); - }, - async () => { - // New version document (delete marker) was created with the right originOp. - sinon.assert.calledWith(metadataswitch.putObjectMD.lastCall, - bucketName, objectKey, sinon.match({ - _data: { originOp: 's3:ObjectRemoved:DeleteMarkerCreated' }, - }), any, any, any); - }, - ], done); + async.series( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + next => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + next => bucketPutVersioning(authInfo, enableVersioningRequest, log, next), + next => objectDelete(authInfo, testDeleteRequest, log, next), + async () => { + const calls = metadataswitch.putObjectMD.getCalls(); + sinon.assert.calledWith( + calls[calls.length - 2], + bucketName, + objectKey, + sinon.match({ + versionId: sinon.match.truthy, + isNull: true, + originOp: 's3:StoreNullVersion', + }), + any, + any, + any, + ); + }, + async () => { + // New version document (delete marker) was created with the right originOp. + sinon.assert.calledWith( + metadataswitch.putObjectMD.lastCall, + bucketName, + objectKey, + sinon.match({ + _data: { originOp: 's3:ObjectRemoved:DeleteMarkerCreated' }, + }), + any, + any, + any, + ); + }, + ], + done, + ); }); }); diff --git a/tests/unit/api/objectDeleteTagging.js b/tests/unit/api/objectDeleteTagging.js index 66ed5a11a2..4e9d751795 100644 --- a/tests/unit/api/objectDeleteTagging.js +++ b/tests/unit/api/objectDeleteTagging.js @@ -6,10 +6,7 @@ const objectPut = require('../../../lib/api/objectPut'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging'); const metadata = require('../../../lib/metadata/wrapper'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const DummyRequest = require('../DummyRequest'); const log = new DummyRequestLogger(); @@ -25,13 +22,16 @@ const testBucketPutRequest = { actionImplicitDenies: false, }; -const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); describe('deleteObjectTagging API', () => { beforeEach(done => { @@ -40,8 +40,7 @@ describe('deleteObjectTagging API', () => { if (err) { return done(err); } - return objectPut(authInfo, testPutObjectRequest, undefined, log, - done); + return objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); @@ -49,22 +48,20 @@ describe('deleteObjectTagging API', () => { it('should delete tag set and update originOp', done => { const taggingUtil = new TaggingConfigTester(); - const testObjectPutTaggingRequest = taggingUtil - .createObjectTaggingRequest('PUT', bucketName, objectName); - const testObjectDeleteTaggingRequest = taggingUtil - .createObjectTaggingRequest('DELETE', bucketName, objectName); - async.waterfall([ - next => objectPutTagging(authInfo, testObjectPutTaggingRequest, log, - err => next(err)), - next => objectDeleteTagging(authInfo, - testObjectDeleteTaggingRequest, log, err => next(err)), - next => metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objectMD) => next(err, objectMD)), - ], (err, objectMD) => { - const uploadedTags = objectMD.tags; - assert.deepStrictEqual(uploadedTags, {}); - assert.strictEqual(objectMD.originOp, 's3:ObjectTagging:Delete'); - return done(); - }); + const testObjectPutTaggingRequest = taggingUtil.createObjectTaggingRequest('PUT', bucketName, objectName); + const testObjectDeleteTaggingRequest = taggingUtil.createObjectTaggingRequest('DELETE', bucketName, objectName); + async.waterfall( + [ + next => objectPutTagging(authInfo, testObjectPutTaggingRequest, log, err => next(err)), + next => objectDeleteTagging(authInfo, testObjectDeleteTaggingRequest, log, err => next(err)), + next => metadata.getObjectMD(bucketName, objectName, {}, log, (err, objectMD) => next(err, objectMD)), + ], + (err, objectMD) => { + const uploadedTags = objectMD.tags; + assert.deepStrictEqual(uploadedTags, {}); + assert.strictEqual(objectMD.originOp, 's3:ObjectTagging:Delete'); + return done(); + }, + ); }); }); diff --git a/tests/unit/api/objectGet.js b/tests/unit/api/objectGet.js index 4ed5d6afef..9d53698ca5 100644 --- a/tests/unit/api/objectGet.js +++ b/tests/unit/api/objectGet.js @@ -8,11 +8,9 @@ const { algorithms } = require('../../../lib/api/apiUtils/integrity/validateChec const { bucketPut } = require('../../../lib/api/bucketPut'); const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); const objectPut = require('../../../lib/api/objectPut'); const objectGet = require('../../../lib/api/objectGet'); const objectPutPart = require('../../../lib/api/objectPutPart'); @@ -32,17 +30,20 @@ describe('objectGet API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-meta-test': 'some metadata', - 'content-length': '12', + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-meta-test': 'some metadata', + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + postBody, + ); }); const correctMD5 = 'be747eb4b75517bf6b3cf7c5fbb62f3a'; @@ -66,19 +67,14 @@ describe('objectGet API', () => { it('should get the object metadata', done => { bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, - log, (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, - log, (err, result, responseMetaHeaders) => { - assert.strictEqual( - responseMetaHeaders[userMetadataKey], - userMetadataValue); - assert.strictEqual(responseMetaHeaders.ETag, - `"${correctMD5}"`); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, result, responseMetaHeaders) => { + assert.strictEqual(responseMetaHeaders[userMetadataKey], userMetadataValue); + assert.strictEqual(responseMetaHeaders.ETag, `"${correctMD5}"`); + done(); }); + }); }); }); @@ -86,25 +82,29 @@ describe('objectGet API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bucket-object-lock-enabled': 'true', }, url: `/${bucketName}`, actionImplicitDenies: false, }; - const createPutDummyRetention = (date, mode) => new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': date, - 'x-amz-object-lock-mode': mode, - 'content-length': '12', - }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + const createPutDummyRetention = (date, mode) => + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': date, + 'x-amz-object-lock-mode': mode, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); const threeDaysMilliSecs = 3 * 24 * 60 * 60 * 1000; const testDate = new Date(Date.now() + threeDaysMilliSecs).toISOString(); @@ -112,204 +112,203 @@ describe('objectGet API', () => { it('should get the object metadata with valid retention info', done => { bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { const request = createPutDummyRetention(testDate, 'GOVERNANCE'); - objectPut(authInfo, request, undefined, - log, (err, headers) => { + objectPut(authInfo, request, undefined, log, (err, headers) => { + assert.ifError(err); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + const req = testGetRequest; + objectGet(authInfo, req, false, log, (err, r, headers) => { assert.ifError(err); + assert.strictEqual(headers['x-amz-object-lock-retain-until-date'], testDate); + assert.strictEqual(headers['x-amz-object-lock-mode'], 'GOVERNANCE'); assert.strictEqual(headers.ETag, `"${correctMD5}"`); - const req = testGetRequest; - objectGet(authInfo, req, false, log, (err, r, headers) => { - assert.ifError(err); - assert.strictEqual( - headers['x-amz-object-lock-retain-until-date'], - testDate); - assert.strictEqual( - headers['x-amz-object-lock-mode'], - 'GOVERNANCE'); - assert.strictEqual(headers.ETag, - `"${correctMD5}"`); - changeObjectLock([{ - bucket: bucketName, - key: objectName, - versionId: headers['x-amz-version-id'], - }], '', done); - }); + changeObjectLock( + [ + { + bucket: bucketName, + key: objectName, + versionId: headers['x-amz-version-id'], + }, + ], + '', + done, + ); }); + }); }); }); - const createPutDummyLegalHold = legalHold => new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-legal-hold': legalHold, - 'content-length': '12', - }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + const createPutDummyLegalHold = legalHold => + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-legal-hold': legalHold, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); const testStatuses = ['ON', 'OFF']; testStatuses.forEach(status => { it(`should get object metadata with legal hold ${status}`, done => { bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { const request = createPutDummyLegalHold(status); - objectPut(authInfo, request, undefined, log, - (err, resHeaders) => { + objectPut(authInfo, request, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, res, headers) => { assert.ifError(err); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, log, - (err, res, headers) => { - assert.ifError(err); - assert.strictEqual( - headers['x-amz-object-lock-legal-hold'], - status); - assert.strictEqual(headers.ETag, - `"${correctMD5}"`); - changeObjectLock([{ + assert.strictEqual(headers['x-amz-object-lock-legal-hold'], status); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + changeObjectLock( + [ + { bucket: bucketName, key: objectName, versionId: headers['x-amz-version-id'], - }], '', done); - }); + }, + ], + '', + done, + ); }); + }); }); }); }); const createPutDummyRetentionAndLegalHold = (date, mode, status) => - new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': date, - 'x-amz-object-lock-mode': mode, - 'x-amz-object-lock-legal-hold': status, - 'content-length': '12', + new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': date, + 'x-amz-object-lock-mode': mode, + 'x-amz-object-lock-legal-hold': status, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${bucketName}/${objectName}`, }, - parsedContentLength: 12, - url: `/${bucketName}/${objectName}`, - }, postBody); + postBody, + ); - it('should get the object metadata with both retention and legal hold', - done => { - bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { - const request = createPutDummyRetentionAndLegalHold( - testDate, 'COMPLIANCE', 'ON'); - objectPut(authInfo, request, undefined, log, - (err, resHeaders) => { - assert.ifError(err); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - const auth = authInfo; - const req = testGetRequest; - objectGet(auth, req, false, log, (err, r, headers) => { - assert.ifError(err); - assert.strictEqual( - headers['x-amz-object-lock-legal-hold'], - 'ON'); - assert.strictEqual( - headers['x-amz-object-lock-retain-until-date'], - testDate); - assert.strictEqual( - headers['x-amz-object-lock-mode'], - 'COMPLIANCE'); - assert.strictEqual(headers.ETag, - `"${correctMD5}"`); - done(); - }); - }); + it('should get the object metadata with both retention and legal hold', done => { + bucketPut(authInfo, testPutBucketRequestObjectLock, log, () => { + const request = createPutDummyRetentionAndLegalHold(testDate, 'COMPLIANCE', 'ON'); + objectPut(authInfo, request, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + const auth = authInfo; + const req = testGetRequest; + objectGet(auth, req, false, log, (err, r, headers) => { + assert.ifError(err); + assert.strictEqual(headers['x-amz-object-lock-legal-hold'], 'ON'); + assert.strictEqual(headers['x-amz-object-lock-retain-until-date'], testDate); + assert.strictEqual(headers['x-amz-object-lock-mode'], 'COMPLIANCE'); + assert.strictEqual(headers.ETag, `"${correctMD5}"`); + done(); + }); }); }); + }); it('should get the object data retrieval info', done => { bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, log, - (err, dataGetInfo) => { - assert.deepStrictEqual(dataGetInfo, - [{ - key: 1, - start: 0, - size: 12, - dataStoreName: 'mem', - dataStoreETag: `1:${correctMD5}`, - }]); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, dataGetInfo) => { + assert.deepStrictEqual(dataGetInfo, [ + { + key: 1, + start: 0, + size: 12, + dataStoreName: 'mem', + dataStoreETag: `1:${correctMD5}`, + }, + ]); + done(); }); + }); }); }); - it('should get the object data retrieval info for an object put by MPU', - done => { - const partBody = Buffer.from('I am a part\n', 'utf8'); - const initiateRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - url: `/${objectName}?uploads`, - actionImplicitDenies: false, - }; - async.waterfall([ + it('should get the object data retrieval info for an object put by MPU', done => { + const partBody = Buffer.from('I am a part\n', 'utf8'); + const initiateRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: `/${objectName}?uploads`, + actionImplicitDenies: false, + }; + async.waterfall( + [ next => bucketPut(authInfo, testPutBucketRequest, log, next), - (corsHeaders, next) => initiateMultipartUpload(authInfo, - initiateRequest, log, next), + (corsHeaders, next) => initiateMultipartUpload(authInfo, initiateRequest, log, next), (result, corsHeaders, next) => parseString(result, next), (json, next) => { - const testUploadId = - json.InitiateMultipartUploadResult.UploadId[0]; + const testUploadId = json.InitiateMultipartUploadResult.UploadId[0]; const partHash = crypto.createHash('md5').update(partBody).digest('hex'); - const partRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - // Part (other than last part) must be at least 5MB - 'content-length': '5242880', - }, - parsedContentLength: 5242880, - url: `/${objectName}?partNumber=1&uploadId` + - `=${testUploadId}`, - query: { - partNumber: '1', - uploadId: testUploadId, + const partRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + // Part (other than last part) must be at least 5MB + 'content-length': '5242880', + }, + parsedContentLength: 5242880, + url: `/${objectName}?partNumber=1&uploadId` + `=${testUploadId}`, + query: { + partNumber: '1', + uploadId: testUploadId, + }, + partHash, }, - partHash, - }, partBody); + partBody, + ); objectPutPart(authInfo, partRequest, undefined, log, () => { next(null, testUploadId, partHash); }); }, (testUploadId, partHash, next) => { - const part2Request = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'host': `${bucketName}.s3.amazonaws.com`, - 'content-length': '12', - }, - parsedContentLength: 12, - url: `/${objectName}?partNumber=2&uploadId=` + - `${testUploadId}`, - query: { - partNumber: '2', - uploadId: testUploadId, + const part2Request = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'content-length': '12', + }, + parsedContentLength: 12, + url: `/${objectName}?partNumber=2&uploadId=` + `${testUploadId}`, + query: { + partNumber: '2', + uploadId: testUploadId, + }, + partHash, }, - partHash, - }, partBody); - objectPutPart(authInfo, part2Request, undefined, - log, () => { - next(null, testUploadId, partHash); - }); + partBody, + ); + objectPutPart(authInfo, part2Request, undefined, log, () => { + next(null, testUploadId, partHash); + }); }, (testUploadId, partHash, next) => { - const completeBody = '' + + const completeBody = + '' + '' + '1' + `"${partHash}"` + @@ -330,19 +329,17 @@ describe('objectGet API', () => { post: completeBody, actionImplicitDenies: false, }; - completeMultipartUpload(authInfo, completeRequest, - log, err => { - next(err, partHash); - }); + completeMultipartUpload(authInfo, completeRequest, log, err => { + next(err, partHash); + }); }, ], (err, partHash) => { assert.ifError(err); - objectGet(authInfo, testGetRequest, false, log, - (err, dataGetInfo) => { + objectGet(authInfo, testGetRequest, false, log, (err, dataGetInfo) => { assert.ifError(err); - assert.deepStrictEqual(dataGetInfo, - [{ + assert.deepStrictEqual(dataGetInfo, [ + { key: 1, dataStoreName: 'mem', dataStoreETag: `1:${partHash}`, @@ -355,42 +352,42 @@ describe('objectGet API', () => { dataStoreETag: `2:${partHash}`, size: 12, start: 5242880, - }]); + }, + ]); done(); }); - }); - }); + }, + ); + }); it('should get a 0 bytes object', done => { const postBody = ''; const correctMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'content-length': '0', - 'x-amz-meta-test': 'some metadata', + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': '0', + 'x-amz-meta-test': 'some metadata', + }, + parsedContentLength: 0, + url: `/${bucketName}/${objectName}`, + partHash: 'd41d8cd98f00b204e9800998ecf8427e', }, - parsedContentLength: 0, - url: `/${bucketName}/${objectName}`, - partHash: 'd41d8cd98f00b204e9800998ecf8427e', - }, postBody); + postBody, + ); bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGet(authInfo, testGetRequest, false, - log, (err, result, responseMetaHeaders) => { - assert.strictEqual(result, null); - assert.strictEqual( - responseMetaHeaders[userMetadataKey], - userMetadataValue); - assert.strictEqual(responseMetaHeaders.ETag, - `"${correctMD5}"`); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGet(authInfo, testGetRequest, false, log, (err, result, responseMetaHeaders) => { + assert.strictEqual(result, null); + assert.strictEqual(responseMetaHeaders[userMetadataKey], userMetadataValue); + assert.strictEqual(responseMetaHeaders.ETag, `"${correctMD5}"`); + done(); }); + }); }); }); @@ -496,55 +493,64 @@ describe('objectGet API', () => { }); }); - it('should reflect the restore header with ongoing-request=false and expiry-date set ' + - 'if the object is restored and not yet expired', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }; - mdColdHelper.putBucketMock(bucketName, null, () => { - const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); - const restoreInfo = objectCustomMDFields.getAmzRestore(); - mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { - objectGet(authInfo, testGetRequest, false, log, (err, res, headers) => { - assert.ifError(err); - assert.ok(res); - assert.strictEqual(headers['x-amz-storage-class'], mdColdHelper.defaultLocation); - const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); - assert.strictEqual(headers['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); - done(); + it( + 'should reflect the restore header with ongoing-request=false and expiry-date set ' + + 'if the object is restored and not yet expired', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }; + mdColdHelper.putBucketMock(bucketName, null, () => { + const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); + const restoreInfo = objectCustomMDFields.getAmzRestore(); + mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { + objectGet(authInfo, testGetRequest, false, log, (err, res, headers) => { + assert.ifError(err); + assert.ok(res); + assert.strictEqual(headers['x-amz-storage-class'], mdColdHelper.defaultLocation); + const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); + assert.strictEqual( + headers['x-amz-restore'], + `ongoing-request="false", expiry-date="${utcDate}"`, + ); + done(); + }); }); }); - }); - }); + }, + ); }); describe('objectGet API - x-amz-checksum-mode', () => { const checksumAlgorithms = [ - { name: 'sha256', header: 'x-amz-checksum-sha256' }, - { name: 'sha1', header: 'x-amz-checksum-sha1' }, - { name: 'crc32', header: 'x-amz-checksum-crc32' }, - { name: 'crc32c', header: 'x-amz-checksum-crc32c' }, + { name: 'sha256', header: 'x-amz-checksum-sha256' }, + { name: 'sha1', header: 'x-amz-checksum-sha1' }, + { name: 'crc32', header: 'x-amz-checksum-crc32' }, + { name: 'crc32c', header: 'x-amz-checksum-crc32c' }, { name: 'crc64nvme', header: 'x-amz-checksum-crc64nvme' }, ]; const expectedDigests = {}; before(done => { - Promise.all(checksumAlgorithms.map(async ({ name }) => { - expectedDigests[name] = await algorithms[name].digest(postBody); - })).then(() => done(), done); + Promise.all( + checksumAlgorithms.map(async ({ name }) => { + expectedDigests[name] = await algorithms[name].digest(postBody); + }), + ).then(() => done(), done); }); beforeEach(() => cleanup()); checksumAlgorithms.forEach(({ name, header }) => { it(`should return ${header} and x-amz-checksum-type when mode is ENABLED`, done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum(name, expectedDigests[name], 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum(name, expectedDigests[name], 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -561,7 +567,8 @@ describe('objectGet API - x-amz-checksum-mode', () => { assert.strictEqual(headers['x-amz-checksum-type'], 'FULL_OBJECT'); done(); }); - })); + }), + ); }); }); @@ -578,17 +585,18 @@ describe('objectGet API - x-amz-checksum-mode', () => { }; objectGet(authInfo, req, false, log, (err, _locator, headers) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(headers[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(headers[header], undefined)); assert.strictEqual(headers['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); it('should not return checksum headers when x-amz-checksum-mode is not set', done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -601,12 +609,12 @@ describe('objectGet API - x-amz-checksum-mode', () => { }; objectGet(authInfo, req, false, log, (err, _locator, headers) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(headers[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(headers[header], undefined)); assert.strictEqual(headers['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); it('should return InvalidArgument when x-amz-checksum-mode is not ENABLED', done => { @@ -625,8 +633,9 @@ describe('objectGet API - x-amz-checksum-mode', () => { }); it('should not return checksum headers when Range header is set', done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -642,17 +651,18 @@ describe('objectGet API - x-amz-checksum-mode', () => { }; objectGet(authInfo, req, false, log, (err, _locator, headers) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(headers[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(headers[header], undefined)); assert.strictEqual(headers['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); it('should not return checksum headers when partNumber is set', done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -665,11 +675,11 @@ describe('objectGet API - x-amz-checksum-mode', () => { }; objectGet(authInfo, req, false, log, (err, _locator, headers) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(headers[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(headers[header], undefined)); assert.strictEqual(headers['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); }); diff --git a/tests/unit/api/objectGetACL.js b/tests/unit/api/objectGetACL.js index 5785a79a6a..3bf2538978 100644 --- a/tests/unit/api/objectGetACL.js +++ b/tests/unit/api/objectGetACL.js @@ -31,7 +31,7 @@ describe('objectGetACL API', () => { bucketName, namespace, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-acl': 'public-read-write', }, url: '/', @@ -48,36 +48,42 @@ describe('objectGetACL API', () => { }; it('should get a canned private ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'private' }, - url: `/${bucketName}/${objectName}`, - post: postBody, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'private' }, + url: `/${bucketName}/${objectName}`, + post: postBody, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + done(); + }, + ); }); - it('should return an error if try to get an ACL ' + - 'for a nonexistent object', done => { + it('should return an error if try to get an ACL ' + 'for a nonexistent object', done => { bucketPut(authInfo, testBucketPutRequest, log, () => { objectGetACL(authInfo, testGetACLRequest, log, err => { assert.strictEqual(err.is.NoSuchKey, true); @@ -87,323 +93,343 @@ describe('objectGetACL API', () => { }); it('should get a canned public-read ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read' }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read' }, + url: `/${bucketName}/${objectName}`, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned public-read-write ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read-write' }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read-write' }, + url: `/${bucketName}/${objectName}`, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .URI[0], constants.publicId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Permission[0], - 'WRITE'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3], undefined); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].URI[0], + constants.publicId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3], undefined); + done(); + }, + ); }); it('should get a canned authenticated-read ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'authenticated-read' }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'authenticated-read' }, + url: `/${bucketName}/${objectName}`, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .URI[0], constants.allAuthedUsersId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], - undefined); - done(); - }); + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].URI[0], + constants.allAuthedUsersId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); + }, + ); }); it('should get a canned bucket-owner-read ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'bucket-owner-read' }, - url: `/${bucketName}/${objectName}`, - post: postBody, - }, postBody); - async.waterfall([ - next => - bucketPut(otherAccountAuthInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut( - authInfo, testPutObjectRequest, undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'bucket-owner-read' }, + url: `/${bucketName}/${objectName}`, + post: postBody, }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], otherAccountCanonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); - }); - - it('should get a canned bucket-owner-full-control ACL', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'bucket-owner-full-control' }, - url: `/${bucketName}/${objectName}`, - calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', - }, postBody); - async.waterfall([ - next => - bucketPut(otherAccountAuthInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + postBody, + ); + async.waterfall( + [ + next => bucketPut(otherAccountAuthInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + otherAccountCanonicalID, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], canonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], otherAccountCanonicalID); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2], undefined); - done(); - }); + ); }); - it('should get specifically set ACLs', done => { - const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', - 'x-amz-grant-read': `uri=${constants.allAuthedUsersId}`, - 'x-amz-grant-write': `uri=${constants.publicId}`, - 'x-amz-grant-read-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2be', - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + - 'f8f8d5218e7cd47ef2bf', + it('should get a canned bucket-owner-full-control ACL', done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'bucket-owner-full-control' }, + url: `/${bucketName}/${objectName}`, + calculatedHash: 'vnR+tLdVF79rPPfF+7YvOg==', }, - url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, - undefined, log, next), - (resHeaders, next) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectGetACL(authInfo, testGetACLRequest, log, next); + postBody, + ); + async.waterfall( + [ + next => bucketPut(otherAccountAuthInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + canonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + otherAccountCanonicalID, + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2], undefined); + done(); }, - (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2be'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[0].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2bf'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[1].Permission[0], - 'FULL_CONTROL'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2bf'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Grantee[0] - .DisplayName[0], 'sampleaccount2@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[2].Permission[0], - 'WRITE_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .ID[0], '79a59df900b949e55d96a1e698fbacedfd6e09d98' + - 'eacf8f8d5218e7cd47ef2be'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Grantee[0] - .DisplayName[0], 'sampleaccount1@sampling.com'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[3].Permission[0], - 'READ_ACP'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4].Grantee[0] - .URI[0], constants.allAuthedUsersId); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[4].Permission[0], - 'READ'); - assert.strictEqual(result.AccessControlPolicy. - AccessControlList[0].Grant[5], - undefined); - done(); - }); + ); }); - const grantsByURI = [ - constants.publicId, - constants.allAuthedUsersId, - ]; - - grantsByURI.forEach(uri => { - it('should get all ACLs when predefined group - ' + - `${uri} is used for multiple grants`, done => { - const testPutObjectRequest = new DummyRequest({ + it('should get specifically set ACLs', done => { + const testPutObjectRequest = new DummyRequest( + { bucketName, namespace, objectKey: objectName, headers: { - 'x-amz-grant-full-control': `uri=${uri}`, - 'x-amz-grant-read': `uri=${uri}`, - 'x-amz-grant-read-acp': `uri=${uri}`, - 'x-amz-grant-write-acp': `uri=${uri}`, + 'x-amz-grant-full-control': + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', + 'x-amz-grant-read': `uri=${constants.allAuthedUsersId}`, + 'x-amz-grant-write': `uri=${constants.publicId}`, + 'x-amz-grant-read-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e698fbacedfd6e09d98eac' + 'f8f8d5218e7cd47ef2bf', }, url: `/${bucketName}/${objectName}`, - }, postBody); - async.waterfall([ - next => bucketPut(authInfo, testBucketPutRequest, - log, next), - (corsHeaders, next) => objectPut(authInfo, - testPutObjectRequest, undefined, log, next), + }, + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), (resHeaders, next) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectGetACL(authInfo, testGetACLRequest, log, next); }, (result, corsHeaders, next) => parseString(result, next), - ], (err, result) => { - assert.ifError(err); - const grants = - result.AccessControlPolicy.AccessControlList[0].Grant; - grants.forEach(grant => { - assert.strictEqual(grant.Permission.length, 1); - assert.strictEqual(grant.Grantee.length, 1); - assert.strictEqual(grant.Grantee[0].URI.length, 1); - assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); - }); + ], + (err, result) => { + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2be', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[0].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2bf', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[1].Permission[0], + 'FULL_CONTROL', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2bf', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[2].Grantee[0].DisplayName[0], + 'sampleaccount2@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[2].Permission[0], 'WRITE_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].ID[0], + '79a59df900b949e55d96a1e698fbacedfd6e09d98' + 'eacf8f8d5218e7cd47ef2be', + ); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[3].Grantee[0].DisplayName[0], + 'sampleaccount1@sampling.com', + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[3].Permission[0], 'READ_ACP'); + assert.strictEqual( + result.AccessControlPolicy.AccessControlList[0].Grant[4].Grantee[0].URI[0], + constants.allAuthedUsersId, + ); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[4].Permission[0], 'READ'); + assert.strictEqual(result.AccessControlPolicy.AccessControlList[0].Grant[5], undefined); done(); - }); + }, + ); + }); + + const grantsByURI = [constants.publicId, constants.allAuthedUsersId]; + + grantsByURI.forEach(uri => { + it('should get all ACLs when predefined group - ' + `${uri} is used for multiple grants`, done => { + const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-grant-full-control': `uri=${uri}`, + 'x-amz-grant-read': `uri=${uri}`, + 'x-amz-grant-read-acp': `uri=${uri}`, + 'x-amz-grant-write-acp': `uri=${uri}`, + }, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); + async.waterfall( + [ + next => bucketPut(authInfo, testBucketPutRequest, log, next), + (corsHeaders, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + (resHeaders, next) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectGetACL(authInfo, testGetACLRequest, log, next); + }, + (result, corsHeaders, next) => parseString(result, next), + ], + (err, result) => { + assert.ifError(err); + const grants = result.AccessControlPolicy.AccessControlList[0].Grant; + grants.forEach(grant => { + assert.strictEqual(grant.Permission.length, 1); + assert.strictEqual(grant.Grantee.length, 1); + assert.strictEqual(grant.Grantee[0].URI.length, 1); + assert.strictEqual(grant.Grantee[0].URI[0], `${uri}`); + }); + done(); + }, + ); }); }); }); diff --git a/tests/unit/api/objectGetLegalHold.js b/tests/unit/api/objectGetLegalHold.js index 910cf12c74..cc2978795f 100644 --- a/tests/unit/api/objectGetLegalHold.js +++ b/tests/unit/api/objectGetLegalHold.js @@ -21,17 +21,19 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); const objectLegalHoldXml = status => - '' + - `${status}`; + '' + `${status}`; const putObjectLegalHoldRequest = status => ({ bucketName, @@ -62,17 +64,17 @@ describe('getObjectLegalHold API', () => { afterEach(cleanup); it('should return InvalidRequest error', done => { - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - err => { - assert.strictEqual(err.is.InvalidRequest, true); - done(); - }); + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, err => { + assert.strictEqual(err.is.InvalidRequest, true); + done(); + }); }); }); describe('with Object Lock enabled on bucket', () => { - const bucketObjectLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjectLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjectLockRequest, log, err => { @@ -83,42 +85,38 @@ describe('getObjectLegalHold API', () => { afterEach(cleanup); - it('should return NoSuchObjectLockConfiguration if no legal hold set', - done => { - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - err => { - assert.strictEqual(err.is.NoSuchObjectLockConfiguration, true); - done(); - }); + it('should return NoSuchObjectLockConfiguration if no legal hold set', done => { + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, err => { + assert.strictEqual(err.is.NoSuchObjectLockConfiguration, true); + done(); }); + }); - it('should get an object\'s legal hold status when OFF', done => { + it("should get an object's legal hold status when OFF", done => { const status = 'OFF'; const request = putObjectLegalHoldRequest(status); objectPutLegalHold(authInfo, request, log, err => { assert.ifError(err); - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - (err, xml) => { - const expectedXml = objectLegalHoldXml(status); - assert.ifError(err); - assert.strictEqual(xml, expectedXml); - done(); - }); + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, (err, xml) => { + const expectedXml = objectLegalHoldXml(status); + assert.ifError(err); + assert.strictEqual(xml, expectedXml); + done(); + }); }); }); - it('should get an object\'s legal hold status when ON', done => { + it("should get an object's legal hold status when ON", done => { const status = 'ON'; const request = putObjectLegalHoldRequest(status); objectPutLegalHold(authInfo, request, log, err => { assert.ifError(err); - objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, - (err, xml) => { - const expectedXml = objectLegalHoldXml(status); - assert.ifError(err); - assert.strictEqual(xml, expectedXml); - done(); - }); + objectGetLegalHold(authInfo, getObjectLegalHoldRequest, log, (err, xml) => { + const expectedXml = objectLegalHoldXml(status); + assert.ifError(err); + assert.strictEqual(xml, expectedXml); + done(); + }); }); }); }); diff --git a/tests/unit/api/objectGetRetention.js b/tests/unit/api/objectGetRetention.js index cd1481f98f..75818e91ce 100644 --- a/tests/unit/api/objectGetRetention.js +++ b/tests/unit/api/objectGetRetention.js @@ -24,15 +24,19 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); -const objectRetentionXml = '' + 'GOVERNANCE' + `${date.toISOString()}` + @@ -74,8 +78,9 @@ describe('getObjectRetention API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjLockRequest, log, err => { @@ -85,19 +90,17 @@ describe('getObjectRetention API', () => { }); afterEach(cleanup); - it('should return NoSuchObjectLockConfiguration if no retention set', - done => { + it('should return NoSuchObjectLockConfiguration if no retention set', done => { objectGetRetention(authInfo, getObjRetRequest, log, err => { assert.strictEqual(err.is.NoSuchObjectLockConfiguration, true); done(); }); }); - it('should get an object\'s retention info', done => { + it("should get an object's retention info", done => { objectPutRetention(authInfo, putObjRetRequest, log, err => { assert.ifError(err); - objectGetRetention(authInfo, getObjRetRequest, log, - (err, xml) => { + objectGetRetention(authInfo, getObjRetRequest, log, (err, xml) => { assert.ifError(err); assert.strictEqual(xml, objectRetentionXml); done(); diff --git a/tests/unit/api/objectGetTagging.js b/tests/unit/api/objectGetTagging.js index b099120fb2..753077b83d 100644 --- a/tests/unit/api/objectGetTagging.js +++ b/tests/unit/api/objectGetTagging.js @@ -4,11 +4,7 @@ const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectGetTagging = require('../../../lib/api/objectGetTagging'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const DummyRequest = require('../DummyRequest'); const log = new DummyRequestLogger(); @@ -24,13 +20,16 @@ const testBucketPutRequest = { actionImplicitDenies: false, }; -const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); describe('getObjectTagging API', () => { beforeEach(done => { @@ -39,8 +38,7 @@ describe('getObjectTagging API', () => { if (err) { return done(err); } - return objectPut(authInfo, testPutObjectRequest, undefined, log, - done); + return objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); @@ -48,17 +46,14 @@ describe('getObjectTagging API', () => { it('should return tags resource', done => { const taggingUtil = new TaggingConfigTester(); - const testObjectPutTaggingRequest = taggingUtil - .createObjectTaggingRequest('PUT', bucketName, objectName); + const testObjectPutTaggingRequest = taggingUtil.createObjectTaggingRequest('PUT', bucketName, objectName); objectPutTagging(authInfo, testObjectPutTaggingRequest, log, err => { if (err) { process.stdout.write(`Err putting object tagging ${err}`); return done(err); } - const testObjectGetTaggingRequest = taggingUtil - .createObjectTaggingRequest('GET', bucketName, objectName); - return objectGetTagging(authInfo, testObjectGetTaggingRequest, log, - (err, xml) => { + const testObjectGetTaggingRequest = taggingUtil.createObjectTaggingRequest('GET', bucketName, objectName); + return objectGetTagging(authInfo, testObjectGetTaggingRequest, log, (err, xml) => { if (err) { process.stdout.write(`Err getting object tagging ${err}`); return done(err); diff --git a/tests/unit/api/objectHead.js b/tests/unit/api/objectHead.js index 9508635c5a..2e7a9a3f2b 100644 --- a/tests/unit/api/objectHead.js +++ b/tests/unit/api/objectHead.js @@ -39,55 +39,61 @@ let testPutObjectRequest; describe('objectHead API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-meta-test': userMetadataValue }, - url: `/${bucketName}/${objectName}`, - calculatedHash: correctMD5, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-meta-test': userMetadataValue }, + url: `/${bucketName}/${objectName}`, + calculatedHash: correctMD5, + }, + postBody, + ); }); - it('should return NotModified if request header ' + - 'includes "if-modified-since" and object ' + - 'not modified since specified time', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-modified-since': laterDate }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return NotModified if request header ' + + 'includes "if-modified-since" and object ' + + 'not modified since specified time', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-modified-since': laterDate }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.NotModified, true); done(); }); }); - }); - }); + }); + }, + ); - it('should return PreconditionFailed if request header ' + - 'includes "if-unmodified-since" and object has ' + - 'been modified since specified time', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-unmodified-since': earlierDate }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return PreconditionFailed if request header ' + + 'includes "if-unmodified-since" and object has ' + + 'been modified since specified time', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-unmodified-since': earlierDate }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.ifError(err); assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { @@ -95,106 +101,118 @@ describe('objectHead API', () => { done(); }); }); - }); - }); + }); + }, + ); - it('should return PreconditionFailed if request header ' + - 'includes "if-match" and ETag of object ' + - 'does not match specified ETag', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-match': incorrectMD5 }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return PreconditionFailed if request header ' + + 'includes "if-match" and ETag of object ' + + 'does not match specified ETag', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-match': incorrectMD5 }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.PreconditionFailed, true); done(); }); }); - }); - }); + }); + }, + ); - it('should return NotModified if request header ' + - 'includes "if-none-match" and ETag of object does ' + - 'match specified ETag', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { 'if-none-match': correctMD5 }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return NotModified if request header ' + + 'includes "if-none-match" and ETag of object does ' + + 'match specified ETag', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { 'if-none-match': correctMD5 }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); objectHead(authInfo, testGetRequest, log, err => { assert.strictEqual(err.is.NotModified, true); done(); }); }); - }); - }); + }); + }, + ); - it('should return Accept-Ranges header if request includes "Range" ' + - 'header with specified range bytes of an object', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { range: 'bytes=1-9' }, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should return Accept-Ranges header if request includes "Range" ' + + 'header with specified range bytes of an object', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { range: 'bytes=1-9' }, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err, null, `Error copying: ${err}`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.strictEqual(res['accept-ranges'], 'bytes'); - done(); + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err, null, `Error copying: ${err}`); + objectHead(authInfo, testGetRequest, log, (err, res) => { + assert.strictEqual(res['accept-ranges'], 'bytes'); + done(); + }); }); }); - }); - }); + }, + ); - it('should return InvalidRequest error when both the Range header and ' + - 'the partNumber query parameter specified', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: { range: 'bytes=1-9' }, - url: `/${bucketName}/${objectName}`, - query: { - partNumber: '1', - }, - actionImplicitDenies: false, - }; + it( + 'should return InvalidRequest error when both the Range header and ' + + 'the partNumber query parameter specified', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: { range: 'bytes=1-9' }, + url: `/${bucketName}/${objectName}`, + query: { + partNumber: '1', + }, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err, null, `Error objectPut: ${err}`); - objectHead(authInfo, testGetRequest, log, err => { - assert.strictEqual(err.is.InvalidRequest, true); - assert.strictEqual(err.description, - 'Cannot specify both Range header and ' + - 'partNumber query parameter.'); - done(); + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err, null, `Error objectPut: ${err}`); + objectHead(authInfo, testGetRequest, log, err => { + assert.strictEqual(err.is.InvalidRequest, true); + assert.strictEqual( + err.description, + 'Cannot specify both Range header and ' + 'partNumber query parameter.', + ); + done(); + }); }); }); - }); - }); + }, + ); it('should return InvalidArgument error if partNumber is nan', done => { const testGetRequest = { @@ -221,27 +239,30 @@ describe('objectHead API', () => { }); }); - it('should not return Accept-Ranges header if request does not include ' + - '"Range" header with specified range bytes of an object', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - actionImplicitDenies: false, - }; + it( + 'should not return Accept-Ranges header if request does not include ' + + '"Range" header with specified range bytes of an object', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + actionImplicitDenies: false, + }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, err => { - assert.strictEqual(err, null, `Error objectPut: ${err}`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.strictEqual(res['accept-ranges'], undefined); - done(); + bucketPut(authInfo, testPutBucketRequest, log, () => { + objectPut(authInfo, testPutObjectRequest, undefined, log, err => { + assert.strictEqual(err, null, `Error objectPut: ${err}`); + objectHead(authInfo, testGetRequest, log, (err, res) => { + assert.strictEqual(res['accept-ranges'], undefined); + done(); + }); }); }); - }); - }); + }, + ); it('should get the object metadata', done => { const testGetRequest = { @@ -254,17 +275,14 @@ describe('objectHead API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.strictEqual(res[userMetadataKey], - userMetadataValue); - assert - .strictEqual(res.ETag, `"${correctMD5}"`); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectHead(authInfo, testGetRequest, log, (err, res) => { + assert.strictEqual(res[userMetadataKey], userMetadataValue); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + done(); }); + }); }); }); @@ -276,18 +294,21 @@ describe('objectHead API', () => { url: `/${bucketName}`, actionImplicitDenies: false, }; - const testPutObjectRequestLock = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'x-amz-object-lock-retain-until-date': '2050-10-10', - 'x-amz-object-lock-mode': 'GOVERNANCE', - 'x-amz-object-lock-legal-hold': 'ON', + const testPutObjectRequestLock = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'x-amz-object-lock-retain-until-date': '2050-10-10', + 'x-amz-object-lock-mode': 'GOVERNANCE', + 'x-amz-object-lock-legal-hold': 'ON', + }, + url: `/${bucketName}/${objectName}`, + calculatedHash: correctMD5, }, - url: `/${bucketName}/${objectName}`, - calculatedHash: correctMD5, - }, postBody); + postBody, + ); const testGetRequest = { bucketName, namespace, @@ -298,31 +319,30 @@ describe('objectHead API', () => { }; bucketPut(authInfo, testPutBucketRequestLock, log, () => { - objectPut(authInfo, testPutObjectRequestLock, undefined, log, - (err, resHeaders) => { + objectPut(authInfo, testPutObjectRequestLock, undefined, log, (err, resHeaders) => { + assert.ifError(err); + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectHead(authInfo, testGetRequest, log, (err, res) => { assert.ifError(err); - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectHead(authInfo, testGetRequest, log, (err, res) => { - assert.ifError(err); - const expectedDate = testPutObjectRequestLock - .headers['x-amz-object-lock-retain-until-date']; - const expectedMode = testPutObjectRequestLock - .headers['x-amz-object-lock-mode']; - assert.ifError(err); - assert.strictEqual( - res['x-amz-object-lock-retain-until-date'], - expectedDate); - assert.strictEqual(res['x-amz-object-lock-mode'], - expectedMode); - assert.strictEqual(res['x-amz-object-lock-legal-hold'], - 'ON'); - changeObjectLock([{ - bucket: bucketName, - key: objectName, - versionId: res['x-amz-version-id'], - }], '', done); - }); + const expectedDate = testPutObjectRequestLock.headers['x-amz-object-lock-retain-until-date']; + const expectedMode = testPutObjectRequestLock.headers['x-amz-object-lock-mode']; + assert.ifError(err); + assert.strictEqual(res['x-amz-object-lock-retain-until-date'], expectedDate); + assert.strictEqual(res['x-amz-object-lock-mode'], expectedMode); + assert.strictEqual(res['x-amz-object-lock-legal-hold'], 'ON'); + changeObjectLock( + [ + { + bucket: bucketName, + key: objectName, + versionId: res['x-amz-version-id'], + }, + ], + '', + done, + ); }); + }); }); }); @@ -396,37 +416,40 @@ describe('objectHead API', () => { }); }); - it('should reflect the restore header with ongoing-request=false and expiry-date set ' + - 'if the object is restored and not yet expired', done => { - const testGetRequest = { - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }; - mdColdHelper.putBucketMock(bucketName, null, () => { - const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); - mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { - objectHead(authInfo, testGetRequest, log, (err, res) => { - const restoreInfo = objectCustomMDFields.getAmzRestore(); - assert.strictEqual(res[userMetadataKey], userMetadataValue); - assert.strictEqual(res.ETag, `"${correctMD5}"`); - assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); - const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); - assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); - // Check we do not leak non-standard fields - assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); - assert.strictEqual(res['x-amz-scal-archive-info'], undefined); - assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined); - assert.strictEqual(res['x-amz-scal-restore-completed-at'], undefined); - assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], undefined); - assert.strictEqual(res['x-amz-scal-owner-id'], undefined); - done(); + it( + 'should reflect the restore header with ongoing-request=false and expiry-date set ' + + 'if the object is restored and not yet expired', + done => { + const testGetRequest = { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }; + mdColdHelper.putBucketMock(bucketName, null, () => { + const objectCustomMDFields = mdColdHelper.getRestoredObjectMD(); + mdColdHelper.putObjectMock(bucketName, objectName, objectCustomMDFields, () => { + objectHead(authInfo, testGetRequest, log, (err, res) => { + const restoreInfo = objectCustomMDFields.getAmzRestore(); + assert.strictEqual(res[userMetadataKey], userMetadataValue); + assert.strictEqual(res.ETag, `"${correctMD5}"`); + assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); + const utcDate = new Date(restoreInfo.getExpiryDate()).toUTCString(); + assert.strictEqual(res['x-amz-restore'], `ongoing-request="false", expiry-date="${utcDate}"`); + // Check we do not leak non-standard fields + assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); + assert.strictEqual(res['x-amz-scal-archive-info'], undefined); + assert.strictEqual(res['x-amz-scal-restore-requested-at'], undefined); + assert.strictEqual(res['x-amz-scal-restore-completed-at'], undefined); + assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], undefined); + assert.strictEqual(res['x-amz-scal-owner-id'], undefined); + done(); + }); }); }); - }); - }); + }, + ); it('should report when transition in progress', done => { const testGetRequest = { @@ -468,8 +491,10 @@ describe('objectHead API', () => { objectHead(authInfo, testGetRequest, log, (err, res) => { assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], true); assert.strictEqual(res['x-amz-scal-transition-in-progress'], true); - assert.strictEqual(res['x-amz-scal-transition-time'], - new Date(objectCustomMDFields.getTransitionTime()).toUTCString()); + assert.strictEqual( + res['x-amz-scal-transition-time'], + new Date(objectCustomMDFields.getTransitionTime()).toUTCString(), + ); assert.strictEqual(res['x-amz-scal-archive-info'], undefined); assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId); done(err); @@ -521,10 +546,11 @@ describe('objectHead API', () => { assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}'); - assert.strictEqual(res['x-amz-scal-restore-requested-at'], - new Date(archive.restoreRequestedAt).toUTCString()); - assert.strictEqual(res['x-amz-scal-restore-requested-days'], - archive.restoreRequestedDays); + assert.strictEqual( + res['x-amz-scal-restore-requested-at'], + new Date(archive.restoreRequestedAt).toUTCString(), + ); + assert.strictEqual(res['x-amz-scal-restore-requested-days'], archive.restoreRequestedDays); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId); done(err); @@ -551,14 +577,19 @@ describe('objectHead API', () => { assert.strictEqual(res['x-amz-meta-scal-s3-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-transition-in-progress'], undefined); assert.strictEqual(res['x-amz-scal-archive-info'], '{"foo":0,"bar":"stuff"}'); - assert.strictEqual(res['x-amz-scal-restore-requested-at'], - new Date(archive.restoreRequestedAt).toUTCString()); - assert.strictEqual(res['x-amz-scal-restore-requested-days'], - archive.restoreRequestedDays); - assert.strictEqual(res['x-amz-scal-restore-completed-at'], - new Date(archive.restoreCompletedAt).toUTCString()); - assert.strictEqual(res['x-amz-scal-restore-will-expire-at'], - new Date(archive.restoreWillExpireAt).toUTCString()); + assert.strictEqual( + res['x-amz-scal-restore-requested-at'], + new Date(archive.restoreRequestedAt).toUTCString(), + ); + assert.strictEqual(res['x-amz-scal-restore-requested-days'], archive.restoreRequestedDays); + assert.strictEqual( + res['x-amz-scal-restore-completed-at'], + new Date(archive.restoreCompletedAt).toUTCString(), + ); + assert.strictEqual( + res['x-amz-scal-restore-will-expire-at'], + new Date(archive.restoreWillExpireAt).toUTCString(), + ); assert.strictEqual(res['x-amz-scal-restore-etag'], mdColdHelper.restoredEtag); assert.strictEqual(res['x-amz-storage-class'], mdColdHelper.defaultLocation); assert.strictEqual(res['x-amz-scal-owner-id'], mdColdHelper.defaultOwnerId); @@ -570,10 +601,10 @@ describe('objectHead API', () => { describe('x-amz-checksum-mode', () => { const checksumAlgorithms = [ - { name: 'sha256', header: 'x-amz-checksum-sha256' }, - { name: 'sha1', header: 'x-amz-checksum-sha1' }, - { name: 'crc32', header: 'x-amz-checksum-crc32' }, - { name: 'crc32c', header: 'x-amz-checksum-crc32c' }, + { name: 'sha256', header: 'x-amz-checksum-sha256' }, + { name: 'sha1', header: 'x-amz-checksum-sha1' }, + { name: 'crc32', header: 'x-amz-checksum-crc32' }, + { name: 'crc32c', header: 'x-amz-checksum-crc32c' }, { name: 'crc64nvme', header: 'x-amz-checksum-crc64nvme' }, ]; @@ -581,15 +612,18 @@ describe('objectHead API', () => { const expectedDigests = {}; before(done => { - Promise.all(checksumAlgorithms.map(async ({ name }) => { - expectedDigests[name] = await algorithms[name].digest(postBody); - })).then(() => done(), done); + Promise.all( + checksumAlgorithms.map(async ({ name }) => { + expectedDigests[name] = await algorithms[name].digest(postBody); + }), + ).then(() => done(), done); }); checksumAlgorithms.forEach(({ name, header }) => { it(`should return ${header} and x-amz-checksum-type when mode is ENABLED`, done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum(name, expectedDigests[name], 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum(name, expectedDigests[name], 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -605,7 +639,8 @@ describe('objectHead API', () => { assert.strictEqual(res['x-amz-checksum-type'], 'FULL_OBJECT'); done(); }); - })); + }), + ); }); }); @@ -621,17 +656,18 @@ describe('objectHead API', () => { }; objectHead(authInfo, req, log, (err, res) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(res[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(res[header], undefined)); assert.strictEqual(res['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); it('should not return checksum headers when x-amz-checksum-mode is not set', done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -643,12 +679,12 @@ describe('objectHead API', () => { }; objectHead(authInfo, req, log, (err, res) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(res[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(res[header], undefined)); assert.strictEqual(res['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); it('should return InvalidArgument when x-amz-checksum-mode is not ENABLED', done => { @@ -666,8 +702,9 @@ describe('objectHead API', () => { }); it('should not return checksum headers when partNumber is set', done => { - const md = new ObjectMD(mdColdHelper.baseMd) - .setChecksum(new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT')); + const md = new ObjectMD(mdColdHelper.baseMd).setChecksum( + new ObjectMDChecksum('sha256', expectedDigests.sha256, 'FULL_OBJECT'), + ); mdColdHelper.putBucketMock(bucketName, null, () => mdColdHelper.putObjectMock(bucketName, objectName, md, () => { const req = { @@ -680,12 +717,12 @@ describe('objectHead API', () => { }; objectHead(authInfo, req, log, (err, res) => { assert.ifError(err); - checksumAlgorithms.forEach(({ header }) => - assert.strictEqual(res[header], undefined)); + checksumAlgorithms.forEach(({ header }) => assert.strictEqual(res[header], undefined)); assert.strictEqual(res['x-amz-checksum-type'], undefined); done(); }); - })); + }), + ); }); }); @@ -694,30 +731,33 @@ describe('objectHead API', () => { name: 'should return content-length of 0 when requesting part 1 of empty object', partNumber: '1', expectedError: null, - expectedContentLength: 0 + expectedContentLength: 0, }, { name: 'should return InvalidRange error when requesting part > 1 of empty object', partNumber: '2', expectedError: 'InvalidRange', - expectedContentLength: undefined - } + expectedContentLength: undefined, + }, ].forEach(testCase => { it(testCase.name, done => { const emptyBody = ''; const emptyMD5 = 'd41d8cd98f00b204e9800998ecf8427e'; - const testPutEmptyObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: { - 'content-length': '0', - 'x-amz-meta-test': userMetadataValue, + const testPutEmptyObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: { + 'content-length': '0', + 'x-amz-meta-test': userMetadataValue, + }, + parsedContentLength: 0, + url: `/${bucketName}/${objectName}`, + calculatedHash: emptyMD5, }, - parsedContentLength: 0, - url: `/${bucketName}/${objectName}`, - calculatedHash: emptyMD5, - }, emptyBody); + emptyBody, + ); const testGetRequest = { bucketName, diff --git a/tests/unit/api/objectPutACL.js b/tests/unit/api/objectPutACL.js index 5ba89d8908..61c033bf1a 100644 --- a/tests/unit/api/objectPutACL.js +++ b/tests/unit/api/objectPutACL.js @@ -6,11 +6,7 @@ const AuthInfo = require('arsenal').auth.AuthInfo; const { bucketPut } = require('../../../lib/api/bucketPut'); const bucketPutPolicy = require('../../../lib/api/bucketPutPolicy'); const constants = require('../../../constants'); -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - AccessControlPolicy } - = require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, AccessControlPolicy } = require('../helpers'); const metadata = require('../metadataswitch'); const objectPut = require('../../../lib/api/objectPut'); const objectPutACL = require('../../../lib/api/objectPutACL'); @@ -20,8 +16,7 @@ const log = new DummyRequestLogger(); const canonicalID = 'accessKey1'; const authInfo = makeAuthInfo(canonicalID); const ownerID = authInfo.getCanonicalID(); -const anotherID = '79a59df900b949e55d96a1e698fba' + - 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; +const anotherID = '79a59df900b949e55d96a1e698fba' + 'cedfd6e09d98eacf8f8d5218e7cd47ef2bf'; const defaultAcpParams = { ownerID, ownerDisplayName: 'OwnerDisplayName', @@ -42,13 +37,16 @@ let testPutObjectRequest; describe('putObjectACL API', () => { beforeEach(() => { cleanup(); - testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, - }, postBody); + testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, + ); }); it('should return an error if invalid canned ACL provided', done => { @@ -63,14 +61,13 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); }); + }); }); }); @@ -86,25 +83,21 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.acl.Canned, - 'public-read-write'); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, 'public-read-write'); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); - it('should set a canned public-read ACL followed by' - + ' a canned authenticated-read ACL', done => { + it('should set a canned public-read ACL followed by' + ' a canned authenticated-read ACL', done => { const testObjACLRequest1 = { bucketName, namespace, @@ -126,30 +119,23 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest1, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.acl.Canned, - 'public-read'); - objectPutACL(authInfo, testObjACLRequest2, log, - err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, - objectName, {}, log, (err, md) => { - assert.strictEqual(md - .acl.Canned, - 'authenticated-read'); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest1, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, 'public-read'); + objectPutACL(authInfo, testObjACLRequest2, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, 'authenticated-read'); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); + }); }); }); }); + }); }); }); @@ -160,8 +146,7 @@ describe('putObjectACL API', () => { objectKey: objectName, headers: { 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="sampleaccount2@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="sampleaccount2@sampling.com"', 'x-amz-grant-read': `uri=${constants.logId}`, 'x-amz-grant-read-acp': `id=${ownerID}`, 'x-amz-grant-write-acp': `id=${anotherID}`, @@ -171,43 +156,34 @@ describe('putObjectACL API', () => { actionImplicitDenies: false, }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(err, null); - const acls = md.acl; - assert.strictEqual(acls.READ[0], - constants.logId); - assert(acls.FULL_CONTROL[0] - .indexOf(ownerID) > -1); - assert(acls.FULL_CONTROL[1] - .indexOf(anotherID) > -1); - assert(acls.READ_ACP[0] - .indexOf(ownerID) > -1); - assert(acls.WRITE_ACP[0] - .indexOf(anotherID) > -1); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); + const acls = md.acl; + assert.strictEqual(acls.READ[0], constants.logId); + assert(acls.FULL_CONTROL[0].indexOf(ownerID) > -1); + assert(acls.FULL_CONTROL[1].indexOf(anotherID) > -1); + assert(acls.READ_ACP[0].indexOf(ownerID) > -1); + assert(acls.WRITE_ACP[0].indexOf(anotherID) > -1); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); - it('should return an error if invalid email ' + - 'provided in ACL header request', done => { + it('should return an error if invalid email ' + 'provided in ACL header request', done => { const testObjACLRequest = { bucketName, namespace, objectKey: objectName, headers: { 'x-amz-grant-full-control': - 'emailaddress="sampleaccount1@sampling.com"' + - ',emailaddress="nonexistentemail@sampling.com"', + 'emailaddress="sampleaccount1@sampling.com"' + ',emailaddress="nonexistentemail@sampling.com"', }, url: `/${bucketName}/${objectName}?acl`, query: { acl: '' }, @@ -215,24 +191,21 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); + done(); }); + }); }); }); it('should set ACLs provided in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); - acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', - 'OwnerDisplayName'); + acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', 'OwnerDisplayName'); acp.addGrantee('Group', constants.publicId, 'READ'); - acp.addGrantee('AmazonCustomerByEmail', 'sampleaccount1@sampling.com', - 'WRITE_ACP'); + acp.addGrantee('AmazonCustomerByEmail', 'sampleaccount1@sampling.com', 'WRITE_ACP'); acp.addGrantee('CanonicalUser', anotherID, 'READ_ACP'); const testObjACLRequest = { bucketName, @@ -246,31 +219,24 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, - log, (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md - .acl.FULL_CONTROL[0], ownerID); - assert.strictEqual(md - .acl.READ[0], constants.publicId); - assert.strictEqual(md - .acl.WRITE_ACP[0], ownerID); - assert.strictEqual(md - .acl.READ_ACP[0], anotherID); - assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.FULL_CONTROL[0], ownerID); + assert.strictEqual(md.acl.READ[0], constants.publicId); + assert.strictEqual(md.acl.WRITE_ACP[0], ownerID); + assert.strictEqual(md.acl.READ_ACP[0], anotherID); + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); - it('should return an error if wrong owner ID ' + - 'provided in ACLs set out in request body', done => { + it('should return an error if wrong owner ID ' + 'provided in ACLs set out in request body', done => { const acp = new AccessControlPolicy({ ownerID: anotherID }); const testObjACLRequest = { bucketName, @@ -284,21 +250,18 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - () => { - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, () => { + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); }); + }); }); }); - it('should ignore if WRITE ACL permission is ' + - 'provided in request body', done => { + it('should ignore if WRITE ACL permission is ' + 'provided in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); - acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', - 'OwnerDisplayName'); + acp.addGrantee('CanonicalUser', ownerID, 'FULL_CONTROL', 'OwnerDisplayName'); acp.addGrantee('Group', constants.publicId, 'WRITE'); const testObjACLRequest = { bucketName, @@ -312,31 +275,25 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.acl.Canned, ''); - assert.strictEqual(md.acl.FULL_CONTROL[0], - ownerID); - assert.strictEqual(md.acl.WRITE, undefined); - assert.strictEqual(md.acl.READ[0], undefined); - assert.strictEqual(md.acl.WRITE_ACP[0], - undefined); - assert.strictEqual(md.acl.READ_ACP[0], - undefined); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.acl.Canned, ''); + assert.strictEqual(md.acl.FULL_CONTROL[0], ownerID); + assert.strictEqual(md.acl.WRITE, undefined); + assert.strictEqual(md.acl.READ[0], undefined); + assert.strictEqual(md.acl.WRITE_ACP[0], undefined); + assert.strictEqual(md.acl.READ_ACP[0], undefined); + done(); }); }); + }); }); }); - it('should return an error if invalid email ' + - 'address provided in ACLs set out in request body', done => { + it('should return an error if invalid email ' + 'address provided in ACLs set out in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); acp.addGrantee('AmazonCustomerByEmail', 'xyz@amazon.com', 'WRITE_ACP'); const testObjACLRequest = { @@ -350,21 +307,18 @@ describe('putObjectACL API', () => { actionImplicitDenies: false, }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.UnresolvableGrantByEmailAddress, true); + done(); }); + }); }); }); - it('should return an error if xml provided does not match s3 ' + - 'scheme for setting ACLs', done => { + it('should return an error if xml provided does not match s3 ' + 'scheme for setting ACLs', done => { const acp = new AccessControlPolicy(defaultAcpParams); acp.addGrantee('AmazonCustomerByEmail', 'xyz@amazon.com', 'WRITE_ACP'); const originalXml = acp.getXml(); @@ -381,14 +335,13 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.MalformedACLError, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.MalformedACLError, true); + done(); }); + }); }); }); @@ -408,24 +361,20 @@ describe('putObjectACL API', () => { actionImplicitDenies: false, }; - bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.MalformedXML, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.MalformedXML, true); + done(); }); + }); }); }); - it('should return an error if invalid group ' + - 'uri provided in ACLs set out in request body', done => { + it('should return an error if invalid group ' + 'uri provided in ACLs set out in request body', done => { const acp = new AccessControlPolicy(defaultAcpParams); - acp.addGrantee('Group', 'http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP', 'WRITE_ACP'); + acp.addGrantee('Group', 'http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP', 'WRITE_ACP'); const testObjACLRequest = { bucketName, namespace, @@ -438,28 +387,24 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); }); + }); }); }); - it('should return an error if invalid group uri ' + - 'provided in ACL header request', done => { + it('should return an error if invalid group uri ' + 'provided in ACL header request', done => { const testObjACLRequest = { bucketName, namespace, objectKey: objectName, headers: { - 'host': 's3.amazonaws.com', - 'x-amz-grant-full-control': - 'uri="http://acs.amazonaws.com/groups/' + - 'global/NOTAVALIDGROUP"', + host: 's3.amazonaws.com', + 'x-amz-grant-full-control': 'uri="http://acs.amazonaws.com/groups/' + 'global/NOTAVALIDGROUP"', }, url: `/${bucketName}/${objectName}?acl`, query: { acl: '' }, @@ -467,14 +412,13 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err.is.InvalidArgument, true); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err.is.InvalidArgument, true); + done(); }); + }); }); }); @@ -484,56 +428,89 @@ describe('putObjectACL API', () => { { headers: { 'x-amz-grant-read': `uri=${constants.logId}` }, type: 'READ' }, { headers: { 'x-amz-grant-read-acp': `id=${ownerID}` }, type: 'READ_ACP' }, { headers: { 'x-amz-grant-write-acp': `id=${anotherID}` }, type: 'WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'ALL' }, - { headers: { - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'READ/READ_ACP/WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'FULL/READ_ACP/WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'FULL/READ/WRITE_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - }, type: 'FULL/READ/READ_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read': `uri=${constants.logId}`, - }, type: 'FULL/READ' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-read-acp': `id=${ownerID}`, - }, type: 'FULL/READ_ACP' }, - { headers: { - 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'FULL/WRITE_ACP' }, - { headers: { - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-read-acp': `id=${ownerID}`, - }, type: 'READ/READ_ACP' }, - { headers: { - 'x-amz-grant-read': `uri=${constants.logId}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'READ/WRITE_ACP' }, - { headers: { - 'x-amz-grant-read-acp': `id=${ownerID}`, - 'x-amz-grant-write-acp': `id=${anotherID}`, - }, type: 'READ_ACP/WRITE_ACP' }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'ALL', + }, + { + headers: { + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'READ/READ_ACP/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'FULL/READ_ACP/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'FULL/READ/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + }, + type: 'FULL/READ/READ_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read': `uri=${constants.logId}`, + }, + type: 'FULL/READ', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-read-acp': `id=${ownerID}`, + }, + type: 'FULL/READ_ACP', + }, + { + headers: { + 'x-amz-grant-full-control': 'emailaddress="sampleaccount1@sampling.com"', + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'FULL/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-read-acp': `id=${ownerID}`, + }, + type: 'READ/READ_ACP', + }, + { + headers: { + 'x-amz-grant-read': `uri=${constants.logId}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'READ/WRITE_ACP', + }, + { + headers: { + 'x-amz-grant-read-acp': `id=${ownerID}`, + 'x-amz-grant-write-acp': `id=${anotherID}`, + }, + type: 'READ_ACP/WRITE_ACP', + }, ].forEach(params => { const { headers, type } = params; it(`should set originOp to s3:ObjectAcl:Put when ACL is changed (${type})`, done => { @@ -546,19 +523,16 @@ describe('putObjectACL API', () => { query: { acl: '' }, }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.originOp, - 's3:ObjectAcl:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.originOp, 's3:ObjectAcl:Put'); + done(); }); }); + }); }); }); }); @@ -574,19 +548,16 @@ describe('putObjectACL API', () => { }; bucketPut(authInfo, testPutBucketRequest, log, () => { - objectPut(authInfo, testPutObjectRequest, undefined, log, - (err, resHeaders) => { - assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); - objectPutACL(authInfo, testObjACLRequest, log, err => { - assert.strictEqual(err, null); - metadata.getObjectMD(bucketName, objectName, {}, - log, (err, md) => { - assert.strictEqual(md.originOp, - 's3:ObjectCreated:Put'); - done(); - }); + objectPut(authInfo, testPutObjectRequest, undefined, log, (err, resHeaders) => { + assert.strictEqual(resHeaders.ETag, `"${correctMD5}"`); + objectPutACL(authInfo, testObjACLRequest, log, err => { + assert.strictEqual(err, null); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + assert.strictEqual(md.originOp, 's3:ObjectCreated:Put'); + done(); }); }); + }); }); }); @@ -608,18 +579,19 @@ describe('putObjectACL API', () => { }; beforeEach(done => { - async.waterfall([ - next => bucketPut(authInfo, testPutBucketRequest, log, next), - (cors, next) => objectPut(authInfo, - testPutObjectRequest, undefined, log, next), - ], err => { - assert.ifError(err); - done(); - }); + async.waterfall( + [ + next => bucketPut(authInfo, testPutBucketRequest, log, next), + (cors, next) => objectPut(authInfo, testPutObjectRequest, undefined, log, next), + ], + err => { + assert.ifError(err); + done(); + }, + ); }); - it('should succeed with a deny on unrelated object as non root', - done => { + it('should succeed with a deny on unrelated object as non root', done => { const bucketPutPolicyRequest = getPolicyRequest({ Version: '2012-10-17', Statement: [ @@ -631,36 +603,38 @@ describe('putObjectACL API', () => { }, ], }); - const testObjACLRequest = Object.assign({ - socket: { - remoteAddress: '1.1.1.1', + const testObjACLRequest = Object.assign( + { + socket: { + remoteAddress: '1.1.1.1', + }, + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read-write' }, + url: `/${bucketName}/${objectName}?acl`, + query: { acl: '' }, + actionImplicitDenies: false, }, - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read-write' }, - url: `/${bucketName}/${objectName}?acl`, - query: { acl: '' }, - actionImplicitDenies: false, - }, requestFix); + requestFix, + ); /** root user doesn't check bucket policy */ const authNotRoot = makeAuthInfo(canonicalID, 'not-root'); - async.waterfall([ - next => bucketPutPolicy(authInfo, - bucketPutPolicyRequest, log, next), - (cors, next) => objectPutACL(authNotRoot, - testObjACLRequest, log, next), - (headers, next) => metadata.getObjectMD(bucketName, - objectName, {}, log, next), - ], (err, md) => { - assert.ifError(err); - assert.strictEqual(md.acl.Canned, 'public-read-write'); - done(); - }); + async.waterfall( + [ + next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), + (cors, next) => objectPutACL(authNotRoot, testObjACLRequest, log, next), + (headers, next) => metadata.getObjectMD(bucketName, objectName, {}, log, next), + ], + (err, md) => { + assert.ifError(err); + assert.strictEqual(md.acl.Canned, 'public-read-write'); + done(); + }, + ); }); - it('should fail with an allow on unrelated object as public', - done => { + it('should fail with an allow on unrelated object as public', done => { const bucketPutPolicyRequest = getPolicyRequest({ Version: '2012-10-17', Statement: [ @@ -672,31 +646,35 @@ describe('putObjectACL API', () => { }, ], }); - const testObjACLRequest = Object.assign({ - socket: { - remoteAddress: '1.1.1.1', + const testObjACLRequest = Object.assign( + { + socket: { + remoteAddress: '1.1.1.1', + }, + bucketName, + namespace, + objectKey: objectName, + headers: { 'x-amz-acl': 'public-read-write' }, + url: `/${bucketName}/${objectName}?acl`, + query: { acl: '' }, + actionImplicitDenies: false, }, - bucketName, - namespace, - objectKey: objectName, - headers: { 'x-amz-acl': 'public-read-write' }, - url: `/${bucketName}/${objectName}?acl`, - query: { acl: '' }, - actionImplicitDenies: false, - }, requestFix); + requestFix, + ); const publicAuth = new AuthInfo({ canonicalID: constants.publicId, }); - async.waterfall([ - next => bucketPutPolicy(authInfo, - bucketPutPolicyRequest, log, next), - (cors, next) => objectPutACL(publicAuth, - testObjACLRequest, log, next), - ], err => { - assert(err instanceof Error); - assert.strictEqual(err.code, errorInstances.AccessDenied.code); - done(); - }); + async.waterfall( + [ + next => bucketPutPolicy(authInfo, bucketPutPolicyRequest, log, next), + (cors, next) => objectPutACL(publicAuth, testObjACLRequest, log, next), + ], + err => { + assert(err instanceof Error); + assert.strictEqual(err.code, errorInstances.AccessDenied.code); + done(); + }, + ); }); }); }); diff --git a/tests/unit/api/objectPutLegalHold.js b/tests/unit/api/objectPutLegalHold.js index 86c41efe89..b8515ec8a6 100644 --- a/tests/unit/api/objectPutLegalHold.js +++ b/tests/unit/api/objectPutLegalHold.js @@ -22,18 +22,19 @@ const putBucketRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); -const objectLegalHoldXml = status => '' + - `${status}` + - ''; +const objectLegalHoldXml = status => + '' + `${status}` + ''; const putLegalHoldReq = status => ({ bucketName, @@ -64,8 +65,9 @@ describe('putObjectLegalHold API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, putBucketRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, putBucketRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjLockRequest, log, err => { @@ -75,11 +77,10 @@ describe('putObjectLegalHold API', () => { }); afterEach(cleanup); - it('should update object\'s metadata with legal hold status', done => { + it("should update object's metadata with legal hold status", done => { objectPutLegalHold(authInfo, putLegalHoldReq('ON'), log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.legalHold, true); return done(); @@ -87,11 +88,10 @@ describe('putObjectLegalHold API', () => { }); }); - it('should update object\'s metadata with legal hold status', done => { + it("should update object's metadata with legal hold status", done => { objectPutLegalHold(authInfo, putLegalHoldReq('OFF'), log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.legalHold, false); return done(); @@ -99,11 +99,10 @@ describe('putObjectLegalHold API', () => { }); }); - it('should set originOp in object\'s metadata to s3:ObjectLegalHold:Put', done => { + it("should set originOp in object's metadata to s3:ObjectLegalHold:Put", done => { objectPutLegalHold(authInfo, putLegalHoldReq('ON'), log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.originOp, 's3:ObjectLegalHold:Put'); return done(); diff --git a/tests/unit/api/objectPutRetention.js b/tests/unit/api/objectPutRetention.js index e0f8b19ae3..1399bca4c7 100644 --- a/tests/unit/api/objectPutRetention.js +++ b/tests/unit/api/objectPutRetention.js @@ -24,39 +24,47 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const putObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); -const objectRetentionXmlGovernance = '' + 'GOVERNANCE' + `${expectedDate}` + ''; -const objectRetentionXmlCompliance = '' + 'COMPLIANCE' + `${expectedDate}` + ''; -const objectRetentionXmlGovernanceLonger = '' + 'GOVERNANCE' + `${moment().add(5, 'days').toISOString()}` + ''; -const objectRetentionXmlGovernanceShorter = '' + 'GOVERNANCE' + `${moment().add(1, 'days').toISOString()}` + ''; -const objectRetentionXmlComplianceShorter = '' + 'COMPLIANCE' + `${moment().add(1, 'days').toISOString()}` + @@ -74,7 +82,7 @@ const putObjRetRequestGovernanceWithHeader = { bucketName, objectKey: objectName, headers: { - 'host': `${bucketName}.s3.amazonaws.com`, + host: `${bucketName}.s3.amazonaws.com`, 'x-amz-bypass-governance-retention': 'true', }, post: objectRetentionXmlGovernance, @@ -134,8 +142,9 @@ describe('putObjectRetention API', () => { }); describe('with Object Lock enabled on bucket', () => { - const bucketObjLockRequest = Object.assign({}, bucketPutRequest, - { headers: { 'x-amz-bucket-object-lock-enabled': 'true' } }); + const bucketObjLockRequest = Object.assign({}, bucketPutRequest, { + headers: { 'x-amz-bucket-object-lock-enabled': 'true' }, + }); beforeEach(done => { bucketPut(authInfo, bucketObjLockRequest, log, err => { @@ -145,11 +154,10 @@ describe('putObjectRetention API', () => { }); afterEach(() => cleanup()); - it('should update an object\'s metadata with retention info', done => { + it("should update an object's metadata with retention info", done => { objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.retentionMode, expectedMode); assert.strictEqual(objMD.retentionDate, expectedDate); @@ -158,11 +166,10 @@ describe('putObjectRetention API', () => { }); }); - it('should set originOp in object\'s metadata to s3:ObjectRetention:Put', done => { + it("should set originOp in object's metadata to s3:ObjectRetention:Put", done => { objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objMD) => { assert.ifError(err); assert.strictEqual(objMD.originOp, 's3:ObjectRetention:Put'); return done(); @@ -190,48 +197,60 @@ describe('putObjectRetention API', () => { }); }); - it('should allow update if the x-amz-bypass-governance-retention header is missing and ' - + 'GOVERNANCE mode is enabled if time is being extended', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernanceLonger, log, err => { + it( + 'should allow update if the x-amz-bypass-governance-retention header is missing and ' + + 'GOVERNANCE mode is enabled if time is being extended', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - done(); + return objectPutRetention(authInfo, putObjRetRequestGovernanceLonger, log, err => { + assert.ifError(err); + done(); + }); }); - }); - }); + }, + ); - it('should disallow update if the x-amz-bypass-governance-retention header is missing and ' - + 'GOVERNANCE mode is enabled', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernanceShorter, log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); + it( + 'should disallow update if the x-amz-bypass-governance-retention header is missing and ' + + 'GOVERNANCE mode is enabled', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { + assert.ifError(err); + return objectPutRetention(authInfo, putObjRetRequestGovernanceShorter, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); + }); }); - }); - }); + }, + ); - it('should allow update if the x-amz-bypass-governance-retention header is missing and ' - + 'GOVERNANCE mode is enabled and the same date is used', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { + it( + 'should allow update if the x-amz-bypass-governance-retention header is missing and ' + + 'GOVERNANCE mode is enabled and the same date is used', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - done(); + return objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { + assert.ifError(err); + done(); + }); }); - }); - }); + }, + ); - it('should allow update if the x-amz-bypass-governance-retention header is present and ' - + 'GOVERNANCE mode is enabled', done => { - objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { - assert.ifError(err); - return objectPutRetention(authInfo, putObjRetRequestGovernanceWithHeader, log, err => { + it( + 'should allow update if the x-amz-bypass-governance-retention header is present and ' + + 'GOVERNANCE mode is enabled', + done => { + objectPutRetention(authInfo, putObjRetRequestGovernance, log, err => { assert.ifError(err); - done(); + return objectPutRetention(authInfo, putObjRetRequestGovernanceWithHeader, log, err => { + assert.ifError(err); + done(); + }); }); - }); - }); + }, + ); }); }); diff --git a/tests/unit/api/objectPutTagging.js b/tests/unit/api/objectPutTagging.js index faa6d9a188..4c0a691fe0 100644 --- a/tests/unit/api/objectPutTagging.js +++ b/tests/unit/api/objectPutTagging.js @@ -3,16 +3,10 @@ const assert = require('assert'); const { bucketPut } = require('../../../lib/api/bucketPut'); const objectPut = require('../../../lib/api/objectPut'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); -const { _validator, parseTagXml } - = require('arsenal').s3middleware.tagging; -const { cleanup, - DummyRequestLogger, - makeAuthInfo, - TaggingConfigTester } - = require('../helpers'); +const { _validator, parseTagXml } = require('arsenal').s3middleware.tagging; +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const metadata = require('../../../lib/metadata/wrapper'); -const { taggingTests } - = require('../../functional/aws-node-sdk/lib/utility/tagging.js'); +const { taggingTests } = require('../../functional/aws-node-sdk/lib/utility/tagging.js'); const DummyRequest = require('../DummyRequest'); const log = new DummyRequestLogger(); @@ -28,13 +22,16 @@ const testBucketPutRequest = { actionImplicitDenies: false, }; -const testPutObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); +const testPutObjectRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey: objectName, + headers: {}, + url: `/${bucketName}/${objectName}`, + }, + postBody, +); function _checkError(err, code, errorName) { assert(err, 'Expected error but found none'); @@ -43,14 +40,15 @@ function _checkError(err, code, errorName) { } function _generateSampleXml(key, value) { - const xml = '' + - '' + - '' + - `${key}` + - `${value}` + - '' + - '' + - ''; + const xml = + '' + + '' + + '' + + `${key}` + + `${value}` + + '' + + '' + + ''; return xml; } @@ -62,24 +60,21 @@ describe('putObjectTagging API', () => { if (err) { return done(err); } - return objectPut(authInfo, testPutObjectRequest, undefined, log, - done); + return objectPut(authInfo, testPutObjectRequest, undefined, log, done); }); }); afterEach(cleanup); - it('should update an object\'s metadata with tags resource and update originOp', done => { + it("should update an object's metadata with tags resource and update originOp", done => { const taggingUtil = new TaggingConfigTester(); - const testObjectPutTaggingRequest = taggingUtil - .createObjectTaggingRequest('PUT', bucketName, objectName); + const testObjectPutTaggingRequest = taggingUtil.createObjectTaggingRequest('PUT', bucketName, objectName); objectPutTagging(authInfo, testObjectPutTaggingRequest, log, err => { if (err) { process.stdout.write(`Err putting object tagging ${err}`); return done(err); } - return metadata.getObjectMD(bucketName, objectName, {}, log, - (err, objectMD) => { + return metadata.getObjectMD(bucketName, objectName, {}, log, (err, objectMD) => { if (err) { process.stdout.write(`Err retrieving object MD ${err}`); return done(err); @@ -95,19 +90,15 @@ describe('putObjectTagging API', () => { describe('PUT object tagging :: helper validation functions ', () => { describe('validateTagStructure ', () => { - it('should return expected true if tag is valid false/undefined if not', - done => { + it('should return expected true if tag is valid false/undefined if not', done => { const tags = [ { tagTest: { Key: ['foo'], Value: ['bar'] }, isValid: true }, { tagTest: { Key: ['foo'] }, isValid: false }, { tagTest: { Value: ['bar'] }, isValid: false }, { tagTest: { Keys: ['foo'], Value: ['bar'] }, isValid: false }, - { tagTest: { Key: ['foo', 'boo'], Value: ['bar'] }, - isValid: false }, - { tagTest: { Key: ['foo'], Value: ['bar', 'boo'] }, - isValid: false }, - { tagTest: { Key: ['foo', 'boo'], Value: ['bar', 'boo'] }, - isValid: false }, + { tagTest: { Key: ['foo', 'boo'], Value: ['bar'] }, isValid: false }, + { tagTest: { Key: ['foo'], Value: ['bar', 'boo'] }, isValid: false }, + { tagTest: { Key: ['foo', 'boo'], Value: ['bar', 'boo'] }, isValid: false }, { tagTest: { Key: ['foo'], Values: ['bar'] }, isValid: false }, { tagTest: { Keys: ['foo'], Values: ['bar'] }, isValid: false }, ]; @@ -125,26 +116,18 @@ describe('PUT object tagging :: helper validation functions ', () => { }); describe('validateXMLStructure ', () => { - it('should return expected true if tag is valid false/undefined ' + - 'if not', done => { + it('should return expected true if tag is valid false/undefined ' + 'if not', done => { const tags = [ - { tagging: { Tagging: { TagSet: [{ Tag: [] }] } }, isValid: - true }, + { tagging: { Tagging: { TagSet: [{ Tag: [] }] } }, isValid: true }, { tagging: { Tagging: { TagSet: [''] } }, isValid: true }, { tagging: { Tagging: { TagSet: [] } }, isValid: false }, { tagging: { Tagging: { TagSet: [{}] } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }] } }, isValid: - false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }] }, - ExtraTagging: 'extratagging' }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: - 'extratagset' } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: - 'extratagset' } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: [], ExtraTag: - 'extratag' }] } }, isValid: false }, - { tagging: { Tagging: { Tagset: [{ Tag: {} }] } }, isValid: - false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }] } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }] }, ExtraTagging: 'extratagging' }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: 'extratagset' } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [] }], ExtraTagset: 'extratagset' } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: [], ExtraTag: 'extratag' }] } }, isValid: false }, + { tagging: { Tagging: { Tagset: [{ Tag: {} }] } }, isValid: false }, ]; for (let i = 0; i < tags.length; i++) { @@ -173,7 +156,9 @@ describe('PUT object tagging :: helper validation functions ', () => { taggingTests.forEach(taggingTest => { it(taggingTest.it, done => { - const { tag: { key, value } } = taggingTest; + const { + tag: { key, value }, + } = taggingTest; const xml = _generateSampleXml(key, value); parseTagXml(xml, log, (err, result) => { if (taggingTest.error) { diff --git a/tests/unit/api/objectRestore.js b/tests/unit/api/objectRestore.js index 28b16248ef..67e3b6a560 100644 --- a/tests/unit/api/objectRestore.js +++ b/tests/unit/api/objectRestore.js @@ -25,38 +25,44 @@ const bucketPutRequest = { actionImplicitDenies: false, }; -const putObjectRequest = new DummyRequest({ - bucketName, - namespace, - objectKey: objectName, - headers: {}, - url: `/${bucketName}/${objectName}`, -}, postBody); - -const objectRestoreXml = '' + `${restoreDays}` + 'Standard' + ''; -const objectRestoreXmlBulkTier = '' + `${restoreDays}` + 'Bulk' + ''; -const objectRestoreXmlExpeditedTier = '' + `${restoreDays}` + 'Expedited' + ''; const objectRestoreRequest = requestXml => ({ - bucketName, - objectKey: objectName, - headers: { host: `${bucketName}.s3.amazonaws.com` }, - post: requestXml, - }); + bucketName, + objectKey: objectName, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + post: requestXml, +}); describe('restoreObject API', () => { before(cleanup); @@ -87,7 +93,7 @@ describe('restoreObject API', () => { }); }); - it('should return NotImplemented error when object restore Tier is \'Bulk\'', done => { + it("should return NotImplemented error when object restore Tier is 'Bulk'", done => { mdColdHelper.putBucketMock(bucketName, null, () => { mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { objectRestore(authInfo, objectRestoreRequest(objectRestoreXmlBulkTier), log, err => { @@ -98,7 +104,7 @@ describe('restoreObject API', () => { }); }); - it('should return NotImplemented error when object restore Tier is \'Expedited\'', done => { + it("should return NotImplemented error when object restore Tier is 'Expedited'", done => { mdColdHelper.putBucketMock(bucketName, null, () => { mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { objectRestore(authInfo, objectRestoreRequest(objectRestoreXmlExpeditedTier), log, err => { @@ -109,48 +115,53 @@ describe('restoreObject API', () => { }); }); - it('should return Accepted and update objectMD ' + - 'while restoring an object from cold storage ' + - 'and the object doesn\'t have a restored copy in bucket', done => { - const testStartTime = new Date(Date.now()); - mdColdHelper.putBucketMock(bucketName, null, () => { - mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { - objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { - assert.ifError(err); - assert.strictEqual(statusCode, 202); - metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { - const testEndTime = new Date(Date.now()); - assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); - assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); - done(); + it( + 'should return Accepted and update objectMD ' + + 'while restoring an object from cold storage ' + + "and the object doesn't have a restored copy in bucket", + done => { + const testStartTime = new Date(Date.now()); + mdColdHelper.putBucketMock(bucketName, null, () => { + mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getArchivedObjectMD(), () => { + objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { + assert.ifError(err); + assert.strictEqual(statusCode, 202); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + const testEndTime = new Date(Date.now()); + assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); + assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); + done(); }); + }); }); }); - }); - }); - - it('should update the expiry time and return OK ' + - 'while restoring an object from cold storage ' + - 'and the object have a restored copy in bucket', done => { - const testStartTime = new Date(Date.now()); - mdColdHelper.putBucketMock(bucketName, null, () => { - mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getRestoredObjectMD(), () => { - objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { - assert.ifError(err); - assert.strictEqual(statusCode, 200); - metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { - const testEndTime = new Date(Date.now()); - assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); - assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); + }, + ); + + it( + 'should update the expiry time and return OK ' + + 'while restoring an object from cold storage ' + + 'and the object have a restored copy in bucket', + done => { + const testStartTime = new Date(Date.now()); + mdColdHelper.putBucketMock(bucketName, null, () => { + mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getRestoredObjectMD(), () => { + objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, (err, statusCode) => { + assert.ifError(err); + assert.strictEqual(statusCode, 200); + metadata.getObjectMD(bucketName, objectName, {}, log, (err, md) => { + const testEndTime = new Date(Date.now()); + assert.strictEqual(md.archive.restoreRequestedDays, restoreDays); + assert.strictEqual(testStartTime < md.archive.restoreRequestedAt < testEndTime, true); done(); }); + }); }); }); - }); - }); + }, + ); - it('should return InvalidObjectState ' + - 'while restoring an expired restored object', () => { + it('should return InvalidObjectState ' + 'while restoring an expired restored object', () => { mdColdHelper.putBucketMock(bucketName, null, () => { mdColdHelper.putObjectMock(bucketName, objectName, mdColdHelper.getExpiredObjectMD(), () => { objectRestore(authInfo, objectRestoreRequest(objectRestoreXml), log, err => { @@ -176,8 +187,11 @@ describe('restoreObject API', () => { try { assert(err, 'Expected an error'); assert.strictEqual(err.is.NoSuchKey, true); - assert.strictEqual(typeof err.customizeDescription, 'function', - 'Error should be from errorInstances which has customizeDescription method'); + assert.strictEqual( + typeof err.customizeDescription, + 'function', + 'Error should be from errorInstances which has customizeDescription method', + ); done(); } catch (assertionError) { done(assertionError); @@ -204,8 +218,11 @@ describe('restoreObject API', () => { try { assert(err, 'Expected an error'); assert.strictEqual(err.is.MethodNotAllowed, true); - assert.strictEqual(typeof err.customizeDescription, 'function', - 'Error should be from errorInstances which has customizeDescription method'); + assert.strictEqual( + typeof err.customizeDescription, + 'function', + 'Error should be from errorInstances which has customizeDescription method', + ); done(); } catch (assertionError) { done(assertionError); diff --git a/tests/unit/api/parseLikeExpression.js b/tests/unit/api/parseLikeExpression.js index 469b6a8df7..20b853c0de 100644 --- a/tests/unit/api/parseLikeExpression.js +++ b/tests/unit/api/parseLikeExpression.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const parseLikeExpression = - require('../../../lib/api/apiUtils/bucket/parseLikeExpression'); +const parseLikeExpression = require('../../../lib/api/apiUtils/bucket/parseLikeExpression'); describe('parseLikeExpression', () => { const tests = [ @@ -29,11 +28,12 @@ describe('parseLikeExpression', () => { output: { $regex: /\//, $options: '' }, }, ]; - tests.forEach(test => it('should return correct MongoDB query object: ' + - `"${test.input}" => ${JSON.stringify(test.output)}`, () => { - const res = parseLikeExpression(test.input); - assert.deepStrictEqual(res, test.output); - })); + tests.forEach(test => + it('should return correct MongoDB query object: ' + `"${test.input}" => ${JSON.stringify(test.output)}`, () => { + const res = parseLikeExpression(test.input); + assert.deepStrictEqual(res, test.output); + }), + ); const badInputTests = [ { input: null, @@ -44,10 +44,10 @@ describe('parseLikeExpression', () => { output: null, }, ]; - badInputTests.forEach(test => it( - 'should return null if input is not a string ' + - `"${test.input}" => ${JSON.stringify(test.output)}`, () => { - const res = parseLikeExpression(test.input); - assert.deepStrictEqual(res, test.output); - })); + badInputTests.forEach(test => + it('should return null if input is not a string ' + `"${test.input}" => ${JSON.stringify(test.output)}`, () => { + const res = parseLikeExpression(test.input); + assert.deepStrictEqual(res, test.output); + }), + ); }); diff --git a/tests/unit/api/serviceGet.js b/tests/unit/api/serviceGet.js index 0804ac103d..054bd191c4 100644 --- a/tests/unit/api/serviceGet.js +++ b/tests/unit/api/serviceGet.js @@ -45,35 +45,33 @@ describe('serviceGet API', () => { url: '/', headers: { host: `${bucketName3}.s3.amazonaws.com` }, }; - async.waterfall([ - function waterfall1(next) { - bucketPut(authInfo, testbucketPutRequest1, log, next); + async.waterfall( + [ + function waterfall1(next) { + bucketPut(authInfo, testbucketPutRequest1, log, next); + }, + function waterfall2(corsHeaders, next) { + bucketPut(authInfo, testbucketPutRequest2, log, next); + }, + function waterfall3(corsHeaders, next) { + bucketPut(authInfo, testbucketPutRequest3, log, next); + }, + function waterfall4(corsHeaders, next) { + serviceGet(authInfo, serviceGetRequest, log, next); + }, + function waterfall4(result, next) { + parseString(result, next); + }, + ], + (err, result) => { + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket.length, 3); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[0].Name[0], bucketName1); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[1].Name[0], bucketName2); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[2].Name[0], bucketName3); + assert.strictEqual(result.ListAllMyBucketsResult.$.xmlns, 'http://s3.amazonaws.com/doc/2006-03-01/'); + done(); }, - function waterfall2(corsHeaders, next) { - bucketPut(authInfo, testbucketPutRequest2, log, next); - }, - function waterfall3(corsHeaders, next) { - bucketPut(authInfo, testbucketPutRequest3, log, next); - }, - function waterfall4(corsHeaders, next) { - serviceGet(authInfo, serviceGetRequest, log, next); - }, - function waterfall4(result, next) { - parseString(result, next); - }, - ], (err, result) => { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket.length, 3); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[0].Name[0], bucketName1); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[1].Name[0], bucketName2); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[2].Name[0], bucketName3); - assert.strictEqual(result.ListAllMyBucketsResult.$.xmlns, - 'http://s3.amazonaws.com/doc/2006-03-01/'); - done(); - }); + ); }); it('should prevent anonymous user from accessing getService API', done => { diff --git a/tests/unit/api/transientBucket.js b/tests/unit/api/transientBucket.js index 9ddf6952db..1890e88576 100644 --- a/tests/unit/api/transientBucket.js +++ b/tests/unit/api/transientBucket.js @@ -14,15 +14,12 @@ const bucketPutWebsite = require('../../../lib/api/bucketPutWebsite'); const bucketDelete = require('../../../lib/api/bucketDelete'); const bucketDeleteCors = require('../../../lib/api/bucketDeleteCors'); const bucketDeleteWebsite = require('../../../lib/api/bucketDeleteWebsite'); -const completeMultipartUpload - = require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const { config } = require('../../../lib/Config'); const constants = require('../../../constants'); const DummyRequest = require('../DummyRequest'); -const initiateMultipartUpload - = require('../../../lib/api/initiateMultipartUpload'); -const { cleanup, createAlteredRequest, DummyRequestLogger, makeAuthInfo } - = require('../helpers'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const { cleanup, createAlteredRequest, DummyRequestLogger, makeAuthInfo } = require('../helpers'); const listMultipartUploads = require('../../../lib/api/listMultipartUploads'); const listParts = require('../../../lib/api/listParts'); const metadata = require('../metadataswitch'); @@ -63,18 +60,15 @@ const serviceGetRequest = { const userBucketOwner = 'admin'; const creationDate = new Date().toJSON(); -const usersBucket = new BucketInfo(usersBucketName, - userBucketOwner, userBucketOwner, creationDate); +const usersBucket = new BucketInfo(usersBucketName, userBucketOwner, userBucketOwner, creationDate); const locationConstraint = 'us-east-1'; describe('transient bucket handling', () => { beforeEach(done => { cleanup(); - const bucketMD = new BucketInfo(bucketName, canonicalID, - authInfo.getAccountDisplayName(), creationDate); + const bucketMD = new BucketInfo(bucketName, canonicalID, authInfo.getAccountDisplayName(), creationDate); bucketMD.addTransientFlag(); - bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), - 'WRITE_ACP'); + bucketMD.setSpecificAcl(otherAccountAuthInfo.getCanonicalID(), 'WRITE_ACP'); bucketMD.setLocationConstraint(locationConstraint); metadata.createBucket(bucketName, bucketMD, log, () => { metadata.createBucket(usersBucketName, usersBucket, log, () => { @@ -83,86 +77,108 @@ describe('transient bucket handling', () => { }); }); - it('putBucket request should complete creation of transient bucket if ' + - 'request is from same account that originally put', done => { - bucketPut(authInfo, baseTestRequest, log, err => { - assert.ifError(err); - serviceGet(authInfo, serviceGetRequest, log, (err, data) => { - parseString(data, (err, result) => { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket.length, 1); - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0].Bucket[0].Name[0], bucketName); - done(); + it( + 'putBucket request should complete creation of transient bucket if ' + + 'request is from same account that originally put', + done => { + bucketPut(authInfo, baseTestRequest, log, err => { + assert.ifError(err); + serviceGet(authInfo, serviceGetRequest, log, (err, data) => { + parseString(data, (err, result) => { + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket.length, 1); + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0].Bucket[0].Name[0], bucketName); + done(); + }); }); }); - }); - }); + }, + ); - it('putBucket request should return error if ' + - 'transient bucket created by different account', done => { + it('putBucket request should return error if ' + 'transient bucket created by different account', done => { bucketPut(otherAccountAuthInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.BucketAlreadyExists, true); - serviceGet(otherAccountAuthInfo, serviceGetRequest, - log, (err, data) => { - parseString(data, (err, result) => { - assert.strictEqual(result.ListAllMyBucketsResult - .Buckets[0], ''); - done(); - }); + serviceGet(otherAccountAuthInfo, serviceGetRequest, log, (err, data) => { + parseString(data, (err, result) => { + assert.strictEqual(result.ListAllMyBucketsResult.Buckets[0], ''); + done(); }); + }); }); }); - it('ACLs from clean up putBucket request should overwrite ACLs from ' + - 'original failed request that resulted in transient state', done => { - const alteredRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPut(authInfo, alteredRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - done(); + it( + 'ACLs from clean up putBucket request should overwrite ACLs from ' + + 'original failed request that resulted in transient state', + done => { + const alteredRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPut(authInfo, alteredRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + done(); + }); }); - }); - }); + }, + ); - it('putBucketACL request should complete creation of transient bucket if ' + - 'request is from same account that originally put', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - putACLRequest.url = '/?acl'; - putACLRequest.query = { acl: '' }; - bucketPutACL(authInfo, putACLRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - done(); + it( + 'putBucketACL request should complete creation of transient bucket if ' + + 'request is from same account that originally put', + done => { + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + putACLRequest.url = '/?acl'; + putACLRequest.query = { acl: '' }; + bucketPutACL(authInfo, putACLRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + done(); + }); }); - }); - }); + }, + ); - it('putBucketACL request should complete creation of transient bucket if ' + - 'request is from another authorized account', done => { - const putACLRequest = createAlteredRequest({ - 'x-amz-acl': 'public-read' }, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutACL(otherAccountAuthInfo, putACLRequest, log, err => { - assert.ifError(err); - metadata.getBucket(bucketName, log, (err, data) => { - assert.strictEqual(data._transient, false); - assert.strictEqual(data._acl.Canned, 'public-read'); - assert.strictEqual(data._owner, authInfo.getCanonicalID()); - done(); + it( + 'putBucketACL request should complete creation of transient bucket if ' + + 'request is from another authorized account', + done => { + const putACLRequest = createAlteredRequest( + { + 'x-amz-acl': 'public-read', + }, + 'headers', + baseTestRequest, + baseTestRequest.headers, + ); + bucketPutACL(otherAccountAuthInfo, putACLRequest, log, err => { + assert.ifError(err); + metadata.getBucket(bucketName, log, (err, data) => { + assert.strictEqual(data._transient, false); + assert.strictEqual(data._acl.Canned, 'public-read'); + assert.strictEqual(data._owner, authInfo.getCanonicalID()); + done(); + }); }); - }); - }); + }, + ); describe('objectPut on a transient bucket', () => { const objName = 'objectName'; @@ -172,10 +188,8 @@ describe('transient bucket handling', () => { }); }); - it('objectPut request should complete creation of transient bucket', - done => { - const setUpRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('objectPut request should complete creation of transient bucket', done => { + const setUpRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); setUpRequest.objectKey = objName; const postBody = Buffer.from('I am a body', 'utf8'); const md5Hash = crypto.createHash('md5'); @@ -186,12 +200,11 @@ describe('transient bucket handling', () => { metadata.getBucket(bucketName, log, (err, data) => { assert.strictEqual(data._transient, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.getObjectMD(bucketName, objName, {}, log, - (err, obj) => { - assert.ifError(err); - assert.strictEqual(obj['content-md5'], etag); - done(); - }); + metadata.getObjectMD(bucketName, objName, {}, log, (err, obj) => { + assert.ifError(err); + assert.strictEqual(obj['content-md5'], etag); + done(); + }); }); }); }); @@ -200,19 +213,15 @@ describe('transient bucket handling', () => { describe('initiateMultipartUpload on a transient bucket', () => { const objName = 'objectName'; after(done => { - metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + - `${bucketName}`, objName, {}, log, () => { - metadata.deleteBucket(`${constants.mpuBucketPrefix}` + - `${bucketName}`, log, () => { - done(); - }); + metadata.deleteObjectMD(`${constants.mpuBucketPrefix}` + `${bucketName}`, objName, {}, log, () => { + metadata.deleteBucket(`${constants.mpuBucketPrefix}` + `${bucketName}`, log, () => { + done(); }); + }); }); - it('initiateMultipartUpload request should complete ' + - 'creation of transient bucket', done => { - const initiateRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('initiateMultipartUpload request should complete ' + 'creation of transient bucket', done => { + const initiateRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); initiateRequest.objectKey = objName; initiateRequest.url = `/${objName}?uploads`; initiateMultipartUpload(authInfo, initiateRequest, log, err => { @@ -220,21 +229,22 @@ describe('transient bucket handling', () => { metadata.getBucket(bucketName, log, (err, data) => { assert.strictEqual(data._transient, false); assert.strictEqual(data._owner, authInfo.getCanonicalID()); - metadata.listObject(`${constants.mpuBucketPrefix}` + - `${bucketName}`, + metadata.listObject( + `${constants.mpuBucketPrefix}` + `${bucketName}`, { prefix: `overview${constants.splitter}${objName}` }, - log, (err, results) => { + log, + (err, results) => { assert.ifError(err); assert.strictEqual(results.Contents.length, 1); done(); - }); + }, + ); }); }); }); }); - it('deleteBucket request should delete transient bucket if ' + - 'request is from owner', done => { + it('deleteBucket request should delete transient bucket if ' + 'request is from owner', done => { bucketDelete(authInfo, baseTestRequest, log, err => { assert.ifError(err); metadata.getBucket(bucketName, log, err => { @@ -244,166 +254,140 @@ describe('transient bucket handling', () => { }); }); - it('deleteBucket request should return error if ' + - 'request is not from owner', done => { - bucketDelete(otherAccountAuthInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.AccessDenied, true); - done(); - }); + it('deleteBucket request should return error if ' + 'request is not from owner', done => { + bucketDelete(otherAccountAuthInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.AccessDenied, true); + done(); + }); }); - it('bucketGet request on transient bucket should return NoSuchBucket' + - 'error', done => { - const bucketGetRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('bucketGet request on transient bucket should return NoSuchBucket' + 'error', done => { + const bucketGetRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); bucketGetRequest.url = `/${bucketName}`; bucketGetRequest.query = {}; - bucketGet(authInfo, bucketGetRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + bucketGet(authInfo, bucketGetRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('bucketGetACL request on transient bucket should return NoSuchBucket' + - 'error', done => { - const bucketGetACLRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('bucketGetACL request on transient bucket should return NoSuchBucket' + 'error', done => { + const bucketGetACLRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); bucketGetACLRequest.url = '/?acl'; bucketGetACLRequest.query = { acl: '' }; - bucketGetACL(authInfo, bucketGetACLRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + bucketGetACL(authInfo, bucketGetACLRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('bucketGetCors request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketGetCors request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketGetCors(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketPutCors request on transient bucket should return ' + - 'NoSuchBucket error', done => { - const bucketPutCorsRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutCorsRequest.post = '' + - 'PUT' + - 'http://www.example.com' + - ''; - bucketPutCorsRequest.headers['content-md5'] = crypto.createHash('md5') - .update(bucketPutCorsRequest.post, 'utf8').digest('base64'); + it('bucketPutCors request on transient bucket should return ' + 'NoSuchBucket error', done => { + const bucketPutCorsRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + bucketPutCorsRequest.post = + '' + + 'PUT' + + 'http://www.example.com' + + ''; + bucketPutCorsRequest.headers['content-md5'] = crypto + .createHash('md5') + .update(bucketPutCorsRequest.post, 'utf8') + .digest('base64'); bucketPutCors(authInfo, bucketPutCorsRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketDeleteCors request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketDeleteCors request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketDeleteCors(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketGetWebsite request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketGetWebsite request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketGetWebsite(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketPutWebsite request on transient bucket should return ' + - 'NoSuchBucket error', done => { - const bucketPutWebsiteRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); - bucketPutWebsiteRequest.post = '' + - 'index.html' + - ''; + it('bucketPutWebsite request on transient bucket should return ' + 'NoSuchBucket error', done => { + const bucketPutWebsiteRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); + bucketPutWebsiteRequest.post = + '' + + 'index.html' + + ''; bucketPutWebsite(authInfo, bucketPutWebsiteRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketDeleteWebsite request on transient bucket should return ' + - 'NoSuchBucket error', done => { + it('bucketDeleteWebsite request on transient bucket should return ' + 'NoSuchBucket error', done => { bucketDeleteWebsite(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('bucketHead request on transient bucket should return NoSuchBucket' + - 'error', done => { - bucketHead(authInfo, baseTestRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + it('bucketHead request on transient bucket should return NoSuchBucket' + 'error', done => { + bucketHead(authInfo, baseTestRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('completeMultipartUpload request on transient bucket should ' + - 'return NoSuchUpload error', done => { - const completeMpuRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('completeMultipartUpload request on transient bucket should ' + 'return NoSuchUpload error', done => { + const completeMpuRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; completeMpuRequest.objectKey = 'objectName'; completeMpuRequest.query = { uploadId }; - completeMultipartUpload(authInfo, completeMpuRequest, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); + completeMultipartUpload(authInfo, completeMpuRequest, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); }); - it('listParts request on transient bucket should ' + - 'return NoSuchUpload error', done => { - const listRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('listParts request on transient bucket should ' + 'return NoSuchUpload error', done => { + const listRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; listRequest.objectKey = 'objectName'; listRequest.query = { uploadId }; - listParts(authInfo, listRequest, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); + listParts(authInfo, listRequest, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); }); describe('multipartDelete request on a transient bucket', () => { - const deleteRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + const deleteRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; deleteRequest.objectKey = 'objectName'; deleteRequest.query = { uploadId }; - const originalLegacyAWSBehavior = - config.locationConstraints[locationConstraint].legacyAwsBehavior; + const originalLegacyAWSBehavior = config.locationConstraints[locationConstraint].legacyAwsBehavior; after(done => { - config.locationConstraints[locationConstraint].legacyAwsBehavior = - originalLegacyAWSBehavior; + config.locationConstraints[locationConstraint].legacyAwsBehavior = originalLegacyAWSBehavior; done(); }); - it('should return NoSuchUpload error if legacyAwsBehavior is enabled', - done => { - config.locationConstraints[locationConstraint]. - legacyAwsBehavior = true; + it('should return NoSuchUpload error if legacyAwsBehavior is enabled', done => { + config.locationConstraints[locationConstraint].legacyAwsBehavior = true; multipartDelete(authInfo, deleteRequest, log, err => { assert.strictEqual(err.is.NoSuchUpload, true); done(); }); }); - it('should return no error if legacyAwsBehavior is not enabled', - done => { + it('should return no error if legacyAwsBehavior is not enabled', done => { config.locationConstraints[locationConstraint].legacyAwsBehavior = false; multipartDelete(authInfo, deleteRequest, log, err => { assert.ifError(err); @@ -412,75 +396,59 @@ describe('transient bucket handling', () => { }); }); - it('objectPutPart request on transient bucket should ' + - 'return NoSuchUpload error', done => { - const putPartRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('objectPutPart request on transient bucket should ' + 'return NoSuchUpload error', done => { + const putPartRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); const uploadId = '5555'; putPartRequest.objectKey = 'objectName'; putPartRequest.query = { uploadId, - partNumber: '1' }; - objectPutPart(authInfo, putPartRequest, undefined, - log, err => { - assert.strictEqual(err.is.NoSuchUpload, true); - done(); - }); + partNumber: '1', + }; + objectPutPart(authInfo, putPartRequest, undefined, log, err => { + assert.strictEqual(err.is.NoSuchUpload, true); + done(); + }); }); - it('list multipartUploads request on transient bucket should ' + - 'return NoSuchBucket error', done => { - const listRequest = createAlteredRequest({}, 'headers', - baseTestRequest, baseTestRequest.headers); + it('list multipartUploads request on transient bucket should ' + 'return NoSuchBucket error', done => { + const listRequest = createAlteredRequest({}, 'headers', baseTestRequest, baseTestRequest.headers); listRequest.query = {}; - listMultipartUploads(authInfo, listRequest, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + listMultipartUploads(authInfo, listRequest, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); + }); }); - it('objectGet request on transient bucket should' + - 'return NoSuchBucket error', - done => { - objectGet(authInfo, baseTestRequest, false, - log, err => { - assert.strictEqual(err.is.NoSuchBucket, true); - done(); - }); + it('objectGet request on transient bucket should' + 'return NoSuchBucket error', done => { + objectGet(authInfo, baseTestRequest, false, log, err => { + assert.strictEqual(err.is.NoSuchBucket, true); + done(); }); + }); - it('objectGetACL request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectGetACL(authInfo, baseTestRequest, - log, err => { + it('objectGetACL request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectGetACL(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('objectHead request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectHead(authInfo, baseTestRequest, - log, err => { + it('objectHead request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectHead(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('objectPutACL request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectPutACL(authInfo, baseTestRequest, - log, err => { + it('objectPutACL request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectPutACL(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); }); - it('objectDelete request on transient bucket should return ' + - 'NoSuchBucket error', done => { - objectDelete(authInfo, baseTestRequest, - log, err => { + it('objectDelete request on transient bucket should return ' + 'NoSuchBucket error', done => { + objectDelete(authInfo, baseTestRequest, log, err => { assert.strictEqual(err.is.NoSuchBucket, true); done(); }); diff --git a/tests/unit/api/utils/metadataMockColdStorage.js b/tests/unit/api/utils/metadataMockColdStorage.js index e2052e5277..22af4d8529 100644 --- a/tests/unit/api/utils/metadataMockColdStorage.js +++ b/tests/unit/api/utils/metadataMockColdStorage.js @@ -4,9 +4,7 @@ const { DummyRequestLogger } = require('../../helpers'); const log = new DummyRequestLogger(); const { BucketInfo, ObjectMD, ObjectMDAmzRestore, ObjectMDArchive } = require('arsenal').models; -const { - LOCATION_NAME_DMF, -} = require('../../../constants'); +const { LOCATION_NAME_DMF } = require('../../../constants'); const defaultLocation = LOCATION_NAME_DMF; const defaultOwnerId = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; @@ -30,7 +28,7 @@ const baseMd = { FULL_CONTROL: [], WRITE_ACP: [], READ: [], - READ_ACP: [] + READ_ACP: [], }, key: 'objectName', location: [ @@ -39,8 +37,8 @@ const baseMd = { size: 11, start: 0, dataStoreName: 'mem', - dataStoreETag: '1:be747eb4b75517bf6b3cf7c5fbb62f3a' - } + dataStoreETag: '1:be747eb4b75517bf6b3cf7c5fbb62f3a', + }, ], isDeleteMarker: false, tags: {}, @@ -53,13 +51,13 @@ const baseMd = { role: '', storageType: '', dataStoreVersionId: '', - isNFS: null + isNFS: null, }, dataStoreName: 'mem', originOp: 's3:ObjectCreated:Put', 'last-modified': '2022-05-10T08:31:51.878Z', 'md-model-version': 5, - 'x-amz-meta-test': 'some metadata' + 'x-amz-meta-test': 'some metadata', }; /** @@ -81,7 +79,8 @@ function putBucketMock(bucketName, location, cb) { null, null, null, - location); + location, + ); return metadata.createBucket(bucketName, bucket, log, cb); } @@ -117,9 +116,11 @@ function getTransitionInProgressObjectMD() { */ function getArchivedObjectMD() { return getTransitionInProgressObjectMD() - .setArchive(new ObjectMDArchive( - { foo: 0, bar: 'stuff' }, // opaque, can be anything... - )) + .setArchive( + new ObjectMDArchive( + { foo: 0, bar: 'stuff' }, // opaque, can be anything... + ), + ) .setDataStoreName(defaultLocation) .setAmzStorageClass(defaultLocation) .setTransitionInProgress(false) @@ -133,14 +134,8 @@ function getArchivedObjectMD() { function getRestoringObjectMD() { const archivedObjectMD = getArchivedObjectMD(); return archivedObjectMD - .setAmzRestore(new ObjectMDAmzRestore( - true, - )) - .setArchive(new ObjectMDArchive( - archivedObjectMD.getArchive().getArchiveInfo(), - new Date(Date.now() - 60), - 5, - )) + .setAmzRestore(new ObjectMDAmzRestore(true)) + .setArchive(new ObjectMDArchive(archivedObjectMD.getArchive().getArchiveInfo(), new Date(Date.now() - 60), 5)) .setOriginOp('s3:ObjectRestore:Post'); } @@ -155,21 +150,20 @@ function getRestoredObjectMD(date) { const expiryDate = date || new Date(restoreDate.getTime() + 1000 * 60 * 60 * 24 * restoreDays); const restoringObjectMD = getRestoringObjectMD(); - const restoreInfo = new ObjectMDAmzRestore( - false, - expiryDate, - ); + const restoreInfo = new ObjectMDAmzRestore(false, expiryDate); restoreInfo['content-md5'] = restoredEtag; return restoringObjectMD .setAmzRestore(restoreInfo) - .setArchive(new ObjectMDArchive( - restoringObjectMD.getArchive().getArchiveInfo(), - new Date(Date.now() - 60000), - restoreDays, - restoreDate, - expiryDate, - )) + .setArchive( + new ObjectMDArchive( + restoringObjectMD.getArchive().getArchiveInfo(), + new Date(Date.now() - 60000), + restoreDays, + restoreDate, + expiryDate, + ), + ) .setDataStoreName('mem') .setOriginOp('s3:ObjectRestore:Completed'); } diff --git a/tests/unit/auth/ChecksumTransform.js b/tests/unit/auth/ChecksumTransform.js index a4265d2b56..291f104bbd 100644 --- a/tests/unit/auth/ChecksumTransform.js +++ b/tests/unit/auth/ChecksumTransform.js @@ -154,15 +154,14 @@ describe('ChecksumTransform', () => { assert.strictEqual(result.details.algorithm, 'crc32'); }); - it('should return TrailerChecksumMalformed when trailer value is not a valid digest for the algo', - async () => { - const stream = new ChecksumTransform('crc32', undefined, true, log); - stream.setExpectedChecksum('x-amz-checksum-crc32', 'not-valid!'); - await drainTransform(stream, [testData]); - const result = stream.validateChecksum(); - assert.strictEqual(result.error, ChecksumError.TrailerChecksumMalformed); - assert.strictEqual(result.details.algorithm, 'crc32'); - }); + it('should return TrailerChecksumMalformed when trailer value is not a valid digest for the algo', async () => { + const stream = new ChecksumTransform('crc32', undefined, true, log); + stream.setExpectedChecksum('x-amz-checksum-crc32', 'not-valid!'); + await drainTransform(stream, [testData]); + const result = stream.validateChecksum(); + assert.strictEqual(result.error, ChecksumError.TrailerChecksumMalformed); + assert.strictEqual(result.details.algorithm, 'crc32'); + }); it('should return XAmzMismatch when trailer value is valid but does not match computed digest', async () => { const stream = new ChecksumTransform('crc32', undefined, true, log); diff --git a/tests/unit/auth/TrailingChecksumTransform.js b/tests/unit/auth/TrailingChecksumTransform.js index 3c408661f2..3b648eaf51 100644 --- a/tests/unit/auth/TrailingChecksumTransform.js +++ b/tests/unit/auth/TrailingChecksumTransform.js @@ -40,7 +40,8 @@ function expectError(inputChunks) { const log = new DummyRequestLogger(); // note this is not the correct checksum in objDataWithTrailingChecksum -const objDataWithTrailingChecksum = '10\r\n01234\r6789abcd\r\n\r\n' + +const objDataWithTrailingChecksum = + '10\r\n01234\r6789abcd\r\n\r\n' + '2\r\n01\r\n' + '1\r\n2\r\n' + 'd\r\n3456789abcdef\r\n' + @@ -74,9 +75,7 @@ describe('TrailingChecksumTransform class', () => { trailingChecksumTransform.on('error', err => { assert.strictEqual(err, null); }); - const chunks = [ - Buffer.from(objDataWithTrailingChecksum), - ]; + const chunks = [Buffer.from(objDataWithTrailingChecksum)]; const chunkedReader = new ChunkedReader(chunks); chunkedReader.pipe(trailingChecksumTransform); const outputChunks = []; @@ -262,24 +261,22 @@ describe('TrailingChecksumTransform class', () => { describe('TrailingChecksumTransform trailer parsing and emitting', () => { describe('happy path', () => { - it('should forward data and emit trailer name and value for single chunk with data and trailer', - async () => { - const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; - const { data, trailers } = await runTransform([input]); - assert.strictEqual(data.toString(), 'hello'); - assert.strictEqual(trailers.length, 1); - assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); - assert.strictEqual(trailers[0].value, 'AAAAAA=='); - }); + it('should forward data and emit trailer name and value for single chunk with data and trailer', async () => { + const input = '5\r\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-crc32'); + assert.strictEqual(trailers[0].value, 'AAAAAA=='); + }); - it('should forward all data and emit trailer once for multiple data chunks followed by trailer', - async () => { - const input = '5\r\nhello\r\n5\r\nworld\r\n0\r\nx-amz-checksum-sha256:AAAAAA==\r\n'; - const { data, trailers } = await runTransform([input]); - assert.strictEqual(data.toString(), 'helloworld'); - assert.strictEqual(trailers.length, 1); - assert.strictEqual(trailers[0].name, 'x-amz-checksum-sha256'); - }); + it('should forward all data and emit trailer once for multiple data chunks followed by trailer', async () => { + const input = '5\r\nhello\r\n5\r\nworld\r\n0\r\nx-amz-checksum-sha256:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([input]); + assert.strictEqual(data.toString(), 'helloworld'); + assert.strictEqual(trailers.length, 1); + assert.strictEqual(trailers[0].name, 'x-amz-checksum-sha256'); + }); it('should forward data chunk containing \\r\\n in payload correctly', async () => { // 7 bytes: h e l \r \n l o @@ -349,15 +346,14 @@ describe('TrailingChecksumTransform class', () => { assert.strictEqual(trailers.length, 1); }); - it('should parse correctly when \\r\\n delimiter after chunk size is split across two chunks', - async () => { - // '5\r' in chunk1, '\nhello\r\n0\r\n...' in chunk2 - const c1 = '5\r'; - const c2 = '\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; - const { data, trailers } = await runTransform([c1, c2]); - assert.strictEqual(data.toString(), 'hello'); - assert.strictEqual(trailers.length, 1); - }); + it('should parse correctly when \\r\\n delimiter after chunk size is split across two chunks', async () => { + // '5\r' in chunk1, '\nhello\r\n0\r\n...' in chunk2 + const c1 = '5\r'; + const c2 = '\nhello\r\n0\r\nx-amz-checksum-crc32:AAAAAA==\r\n'; + const { data, trailers } = await runTransform([c1, c2]); + assert.strictEqual(data.toString(), 'hello'); + assert.strictEqual(trailers.length, 1); + }); it('should emit trailer correctly when trailer line is split across two input chunks', async () => { const c1 = '5\r\nhello\r\n0\r\nx-amz-checksum-'; @@ -405,12 +401,11 @@ describe('TrailingChecksumTransform class', () => { assert.deepStrictEqual(err, errors.IncompleteBody); }); - it('should return IncompleteBody error when stream ends after zero-chunk with partial trailer content', - async () => { - // zero-chunk received, trailer starts but no \r\n terminator - const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==']); - assert.deepStrictEqual(err, errors.IncompleteBody); - }); + it('should return IncompleteBody error when stream ends after zero-chunk with partial trailer content', async () => { + // zero-chunk received, trailer starts but no \r\n terminator + const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); it('should return no error when stream ends after zero-chunk with no trailer content', async () => { // only '0\r\n' — readingTrailer=true, trailerBuffer empty → no error @@ -424,24 +419,21 @@ describe('TrailingChecksumTransform class', () => { assert.deepStrictEqual(err, errors.IncompleteBody); }); - it('should return IncompleteBody when stream ends in READ_CHUNK_DATA_CR (all data read, no trailing CR)', - async () => { - // exactly 5 bytes of data — state is READ_CHUNK_DATA_CR when stream ends - const err = await expectError(['5\r\nhello']); - assert.deepStrictEqual(err, errors.IncompleteBody); - }); + it('should return IncompleteBody when stream ends in READ_CHUNK_DATA_CR (all data read, no trailing CR)', async () => { + // exactly 5 bytes of data — state is READ_CHUNK_DATA_CR when stream ends + const err = await expectError(['5\r\nhello']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); - it('should return IncompleteBody when stream ends in READ_CHUNK_DATA_LF (trailing CR seen, no LF)', - async () => { - const err = await expectError(['5\r\nhello\r']); - assert.deepStrictEqual(err, errors.IncompleteBody); - }); + it('should return IncompleteBody when stream ends in READ_CHUNK_DATA_LF (trailing CR seen, no LF)', async () => { + const err = await expectError(['5\r\nhello\r']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); - it('should return IncompleteBody when stream ends in READ_TRAILER_CHECKSUM_LF (trailing CR seen, no LF)', - async () => { - const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==\r']); - assert.deepStrictEqual(err, errors.IncompleteBody); - }); + it('should return IncompleteBody when stream ends in READ_TRAILER_CHECKSUM_LF (trailing CR seen, no LF)', async () => { + const err = await expectError(['0\r\nx-amz-checksum-crc32:AAAAAA==\r']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); }); describe('_transform error cases', () => { @@ -475,11 +467,10 @@ describe('TrailingChecksumTransform class', () => { assert.deepStrictEqual(err, errors.IncompleteBody); }); - it('should return IncompleteBody error for trailer line with colon at position 0 (empty name)', - async () => { - const err = await expectError(['0\r\n:value\r\n']); - assert.deepStrictEqual(err, errors.IncompleteBody); - }); + it('should return IncompleteBody error for trailer line with colon at position 0 (empty name)', async () => { + const err = await expectError(['0\r\n:value\r\n']); + assert.deepStrictEqual(err, errors.IncompleteBody); + }); it('should return InvalidArgument for empty chunk size (CR at start of chunk header)', async () => { // bare \r\n as a chunk header — bufferOffset=0 when CR seen diff --git a/tests/unit/auth/V4Transform.js b/tests/unit/auth/V4Transform.js index daad1138df..419ba1e10d 100644 --- a/tests/unit/auth/V4Transform.js +++ b/tests/unit/auth/V4Transform.js @@ -7,8 +7,7 @@ const { DummyRequestLogger } = require('../helpers'); const log = new DummyRequestLogger(); const streamingV4Params = { accessKey: 'accessKey1', - signatureFromRequest: '2b8637632a997e06ee7b6c85d7' + - '147d2025e8f04d4374f4d7d7320de1618c7509', + signatureFromRequest: '2b8637632a997e06ee7b6c85d7' + '147d2025e8f04d4374f4d7d7320de1618c7509', region: 'us-east-1', scopeDate: '20170516', timestamp: '20170516T204738Z', @@ -33,15 +32,10 @@ describe('V4Transform class', () => { const v4Transform = new V4Transform(streamingV4Params, log, err => { assert.strictEqual(err, null); }); - const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + - '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; - const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + - '4cc8b7991491806689da0395c8bd0231e48\r\n'; - const chunks = [ - Buffer.from(filler1), - Buffer.from(filler2), - null, - ]; + const filler1 = + '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; + const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + '4cc8b7991491806689da0395c8bd0231e48\r\n'; + const chunks = [Buffer.from(filler1), Buffer.from(filler2), null]; const authMe = new AuthMe(chunks); authMe.pipe(v4Transform); v4Transform.on('finish', () => { @@ -54,15 +48,10 @@ describe('V4Transform class', () => { assert(err); done(); }); - const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + - '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; - const filler2 = '0;chunk-signature=baadc0debaadc0debaadc0debaadc0de' + - 'baadc0debaadc0debaadc0debaadc0de\r\n'; - const chunks = [ - Buffer.from(filler1), - Buffer.from(filler2), - null, - ]; + const filler1 = + '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; + const filler2 = '0;chunk-signature=baadc0debaadc0debaadc0debaadc0de' + 'baadc0debaadc0debaadc0debaadc0de\r\n'; + const chunks = [Buffer.from(filler1), Buffer.from(filler2), null]; const authMe = new AuthMe(chunks); authMe.pipe(v4Transform); }); @@ -71,17 +60,11 @@ describe('V4Transform class', () => { const v4Transform = new V4Transform(streamingV4Params, log, () => { assert(false); }); - const filler1 = '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + - '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; - const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + - '4cc8b7991491806689da0395c8bd0231e48\r\n'; + const filler1 = + '8;chunk-signature=51d2511f7c6887907dff20474d8db6' + '7d557e5f515a6fa6a8466bb12f8833bcca\r\ncontents\r\n'; + const filler2 = '0;chunk-signature=c0eac24b7ce72141ec077df9753db' + '4cc8b7991491806689da0395c8bd0231e48\r\n'; const filler3 = '\r\n'; - const chunks = [ - Buffer.from(filler1), - Buffer.from(filler2), - Buffer.from(filler3), - null, - ]; + const chunks = [Buffer.from(filler1), Buffer.from(filler2), Buffer.from(filler3), null]; const authMe = new AuthMe(chunks); authMe.pipe(v4Transform); v4Transform.on('finish', () => { diff --git a/tests/unit/auth/in_memory/backend.js b/tests/unit/auth/in_memory/backend.js index bc31066158..102f9ddabe 100644 --- a/tests/unit/auth/in_memory/backend.js +++ b/tests/unit/auth/in_memory/backend.js @@ -1,7 +1,6 @@ const assert = require('assert'); -const { buildAuthDataAccount } = - require('../../../../lib/auth/in_memory/builder'); +const { buildAuthDataAccount } = require('../../../../lib/auth/in_memory/builder'); const fakeAccessKey = 'fakeaccesskey'; const fakeSecretKey = 'fakesecretkey'; @@ -16,18 +15,20 @@ function getFirstAndOnlyAccount(authdata) { } describe('buildAuthDataAccount function', () => { - it('should return authdata with the default user name if no user ' + - 'name provided', () => { - const authdata = buildAuthDataAccount(fakeAccessKey, fakeSecretKey, - fakeCanonicalId, fakeServiceName); + it('should return authdata with the default user name if no user ' + 'name provided', () => { + const authdata = buildAuthDataAccount(fakeAccessKey, fakeSecretKey, fakeCanonicalId, fakeServiceName); const firstAccount = getFirstAndOnlyAccount(authdata); assert.strictEqual(firstAccount.name, defaultUserName); }); - it('should return authdata with the user name that has been ' + - 'provided', () => { - const authdata = buildAuthDataAccount(fakeAccessKey, fakeSecretKey, - fakeCanonicalId, fakeServiceName, fakeUserName); + it('should return authdata with the user name that has been ' + 'provided', () => { + const authdata = buildAuthDataAccount( + fakeAccessKey, + fakeSecretKey, + fakeCanonicalId, + fakeServiceName, + fakeUserName, + ); const firstAccount = getFirstAndOnlyAccount(authdata); assert.strictEqual(firstAccount.name, fakeUserName); }); diff --git a/tests/unit/auth/permissionChecks.js b/tests/unit/auth/permissionChecks.js index 69aaa5b9f9..703422958d 100644 --- a/tests/unit/auth/permissionChecks.js +++ b/tests/unit/auth/permissionChecks.js @@ -25,35 +25,50 @@ describe('checkBucketAcls', () => { { description: 'should return true if bucket owner matches canonicalID', input: { - bucketAcl: {}, requestType: 'anyType', canonicalID: 'ownerId', mainApiCall: 'anyApiCall', + bucketAcl: {}, + requestType: 'anyType', + canonicalID: 'ownerId', + mainApiCall: 'anyApiCall', }, expected: true, }, { description: 'should return true for objectGetTagging when mainApiCall is objectGet', input: { - bucketAcl: {}, requestType: 'objectGetTagging', canonicalID: 'anyId', mainApiCall: 'objectGet', + bucketAcl: {}, + requestType: 'objectGetTagging', + canonicalID: 'anyId', + mainApiCall: 'objectGet', }, expected: true, }, { description: 'should return true for objectPutTagging when mainApiCall is objectPut', input: { - bucketAcl: {}, requestType: 'objectPutTagging', canonicalID: 'anyId', mainApiCall: 'objectPut', + bucketAcl: {}, + requestType: 'objectPutTagging', + canonicalID: 'anyId', + mainApiCall: 'objectPut', }, expected: true, }, { description: 'should return true for objectPutLegalHold when mainApiCall is objectPut', input: { - bucketAcl: {}, requestType: 'objectPutLegalHold', canonicalID: 'anyId', mainApiCall: 'objectPut', + bucketAcl: {}, + requestType: 'objectPutLegalHold', + canonicalID: 'anyId', + mainApiCall: 'objectPut', }, expected: true, }, { description: 'should return true for objectPutRetention when mainApiCall is objectPut', input: { - bucketAcl: {}, requestType: 'objectPutRetention', canonicalID: 'anyId', mainApiCall: 'objectPut', + bucketAcl: {}, + requestType: 'objectPutRetention', + canonicalID: 'anyId', + mainApiCall: 'objectPut', }, expected: true, }, @@ -62,7 +77,10 @@ describe('checkBucketAcls', () => { input: { bucketAcl: { Canned: 'public-read-write', - }, requestType: 'initiateMultipartUpload', canonicalID: 'any', mainApiCall: 'initiateMultipartUpload', + }, + requestType: 'initiateMultipartUpload', + canonicalID: 'any', + mainApiCall: 'initiateMultipartUpload', }, expected: true, }, @@ -71,7 +89,10 @@ describe('checkBucketAcls', () => { input: { bucketAcl: { Canned: 'public-read-write', - }, requestType: 'objectPutPart', canonicalID: 'any', mainApiCall: 'objectPutPart', + }, + requestType: 'objectPutPart', + canonicalID: 'any', + mainApiCall: 'objectPutPart', }, expected: true, }, @@ -80,7 +101,10 @@ describe('checkBucketAcls', () => { input: { bucketAcl: { Canned: 'public-read-write', - }, requestType: 'completeMultipartUpload', canonicalID: 'any', mainApiCall: 'completeMultipartUpload', + }, + requestType: 'completeMultipartUpload', + canonicalID: 'any', + mainApiCall: 'completeMultipartUpload', }, expected: true, }, @@ -240,8 +264,12 @@ describe('checkBucketAcls', () => { // Mock the bucket based on the test scenario's input mockBucket.getAcl = () => scenario.input.bucketAcl; - const result = checkBucketAcls(mockBucket, - scenario.input.requestType, scenario.input.canonicalID, scenario.input.mainApiCall); + const result = checkBucketAcls( + mockBucket, + scenario.input.requestType, + scenario.input.canonicalID, + scenario.input.mainApiCall, + ); assert.strictEqual(result, scenario.expected); }); }); @@ -255,7 +283,7 @@ describe('checkObjectAcls', () => { }; const mockObjectMD = { 'owner-id': 'objectOwnerId', - 'acl': { + acl: { Canned: '', FULL_CONTROL: [], READ: [], @@ -266,42 +294,73 @@ describe('checkObjectAcls', () => { }; it('should return true if request type is in bucketOwnerActions and bucket owner matches canonicalID', () => { - assert.strictEqual(checkObjectAcls(mockBucket, mockObjectMD, bucketOwnerActions[0], - 'bucketOwnerId', false, false, 'anyApiCall'), true); + assert.strictEqual( + checkObjectAcls( + mockBucket, + mockObjectMD, + bucketOwnerActions[0], + 'bucketOwnerId', + false, + false, + 'anyApiCall', + ), + true, + ); }); it('should return true if objectMD owner matches canonicalID', () => { - assert.strictEqual(checkObjectAcls(mockBucket, mockObjectMD, 'anyType', - 'objectOwnerId', false, false, 'anyApiCall'), true); + assert.strictEqual( + checkObjectAcls(mockBucket, mockObjectMD, 'anyType', 'objectOwnerId', false, false, 'anyApiCall'), + true, + ); }); it('should return true for objectGetTagging when mainApiCall is objectGet and conditions met', () => { - assert.strictEqual(checkObjectAcls(mockBucket, mockObjectMD, 'objectGetTagging', - 'anyIdNotPublic', true, true, 'objectGet'), true); + assert.strictEqual( + checkObjectAcls(mockBucket, mockObjectMD, 'objectGetTagging', 'anyIdNotPublic', true, true, 'objectGet'), + true, + ); }); it('should return false if no acl provided in objectMD', () => { const objMDWithoutAcl = Object.assign({}, mockObjectMD); delete objMDWithoutAcl.acl; - assert.strictEqual(checkObjectAcls(mockBucket, objMDWithoutAcl, 'anyType', - 'anyId', false, false, 'anyApiCall'), false); + assert.strictEqual( + checkObjectAcls(mockBucket, objMDWithoutAcl, 'anyType', 'anyId', false, false, 'anyApiCall'), + false, + ); }); const tests = [ { - acl: 'public-read', reqType: 'objectGet', id: 'anyIdNotPublic', expected: true, + acl: 'public-read', + reqType: 'objectGet', + id: 'anyIdNotPublic', + expected: true, }, { - acl: 'public-read-write', reqType: 'objectGet', id: 'anyIdNotPublic', expected: true, + acl: 'public-read-write', + reqType: 'objectGet', + id: 'anyIdNotPublic', + expected: true, }, { - acl: 'authenticated-read', reqType: 'objectGet', id: 'anyIdNotPublic', expected: true, + acl: 'authenticated-read', + reqType: 'objectGet', + id: 'anyIdNotPublic', + expected: true, }, { - acl: 'bucket-owner-read', reqType: 'objectGet', id: 'bucketOwnerId', expected: true, + acl: 'bucket-owner-read', + reqType: 'objectGet', + id: 'bucketOwnerId', + expected: true, }, { - acl: 'bucket-owner-full-control', reqType: 'objectGet', id: 'bucketOwnerId', expected: true, + acl: 'bucket-owner-full-control', + reqType: 'objectGet', + id: 'bucketOwnerId', + expected: true, }, { aclList: ['someId', 'anyIdNotPublic'], @@ -323,19 +382,31 @@ describe('checkObjectAcls', () => { { reqType: 'completeMultipartUpload', id: 'anyId', expected: true }, { reqType: 'objectDelete', id: 'anyId', expected: true }, { - aclList: ['anyId'], aclField: 'FULL_CONTROL', reqType: 'objectPutACL', id: 'anyId', expected: true, + aclList: ['anyId'], + aclField: 'FULL_CONTROL', + reqType: 'objectPutACL', + id: 'anyId', + expected: true, }, { - aclList: ['anyId'], aclField: 'FULL_CONTROL', reqType: 'objectGetACL', id: 'anyId', expected: true, + aclList: ['anyId'], + aclField: 'FULL_CONTROL', + reqType: 'objectGetACL', + id: 'anyId', + expected: true, }, { - acl: '', reqType: 'objectGet', id: 'randomId', expected: false, + acl: '', + reqType: 'objectGet', + id: 'randomId', + expected: false, }, ]; tests.forEach(test => { - it(`should return ${test.expected} for ${test.reqType} with ACL as ${test.acl - || (`${test.aclField}:${JSON.stringify(test.aclList)}`)}`, () => { + it(`should return ${test.expected} for ${test.reqType} with ACL as ${ + test.acl || `${test.aclField}:${JSON.stringify(test.aclList)}` + }`, () => { if (test.acl) { mockObjectMD.acl.Canned = test.acl; } else if (test.aclList && test.aclField) { @@ -360,112 +431,123 @@ describe('validatePolicyConditions', () => { { description: 'Should return null if conditions have a valid IP address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '192.168.1.1/24' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '192.168.1.1/24' }, + }, }, - }], + ], }, expected: null, }, { - description: 'Should return "Invalid IP address in Conditions" ' + - 'if conditions have an invalid IP address', + description: + 'Should return "Invalid IP address in Conditions" ' + 'if conditions have an invalid IP address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '123' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '123' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { - description: 'Should return "Policy has an invalid condition key" if a' + - ' condition key does not start with \'aws:\' and is not recognized', + description: + 'Should return "Policy has an invalid condition key" if a' + + " condition key does not start with 'aws:' and is not recognized", inputPolicy: { - Statement: [{ - Condition: { - NotARealCondition: { 's3:prefix': 'something' }, + Statement: [ + { + Condition: { + NotARealCondition: { 's3:prefix': 'something' }, + }, }, - }], + ], }, expected: 'Policy has an invalid condition key', }, { - description: 'Should return null if a statement in the policy does not contain a \'Condition\' block', + description: "Should return null if a statement in the policy does not contain a 'Condition' block", inputPolicy: { Statement: [{}], }, expected: null, }, { - description: 'Should return a relevant error message ' + - 'if the condition value is an empty string', + description: 'Should return a relevant error message ' + 'if the condition value is an empty string', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { description: 'Should accept arrays of IPs', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { - 'aws:SourceIp': [ - '10.0.11.0/24', - '10.0.1.0/24', - ], + Statement: [ + { + Condition: { + IpAddress: { + 'aws:SourceIp': ['10.0.11.0/24', '10.0.1.0/24'], + }, }, }, - }], + ], }, expected: null, }, { description: 'Should return relevant error if one of the IPs in the array is invalid', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { - 'aws:SourceIp': [ - '10.0.11.0/24', - '123', - ], + Statement: [ + { + Condition: { + IpAddress: { + 'aws:SourceIp': ['10.0.11.0/24', '123'], + }, }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { description: 'Should not return error if array value in IP condition is empty', // this is AWS behavior inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { - 'aws:SourceIp': [], + Statement: [ + { + Condition: { + IpAddress: { + 'aws:SourceIp': [], + }, }, }, - }], + ], }, expected: null, }, { - description: 'Should return null or a relevant error message ' + - 'if multiple conditions are provided in a single statement', + description: + 'Should return null or a relevant error message ' + + 'if multiple conditions are provided in a single statement', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '192.168.1.1' }, - NotARealCondition: { 's3:prefix': 'something' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '192.168.1.1' }, + NotARealCondition: { 's3:prefix': 'something' }, + }, }, - }], + ], }, expected: 'Policy has an invalid condition key', }, @@ -490,34 +572,41 @@ describe('validatePolicyConditions', () => { { description: 'Should return null if conditions have a valid IPv6 address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:0000:8a2e:0370:7334' }, + }, }, - }], + ], }, expected: null, }, { description: 'Should return "Invalid IP address in Conditions" if conditions have an invalid IPv6 address', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:XYZZ:8a2e:0370:7334' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '2001:0db8:85a3:0000:XYZZ:8a2e:0370:7334' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, { - description: 'Should return "Invalid IP address in Conditions" if conditions' - + ' have an IPv6 address with unusual and invalid notation', + description: + 'Should return "Invalid IP address in Conditions" if conditions' + + ' have an IPv6 address with unusual and invalid notation', inputPolicy: { - Statement: [{ - Condition: { - IpAddress: { 'aws:SourceIp': '2001::85a3::8a2e' }, + Statement: [ + { + Condition: { + IpAddress: { 'aws:SourceIp': '2001::85a3::8a2e' }, + }, }, - }], + ], }, expected: 'Invalid IP address in Conditions', }, diff --git a/tests/unit/bucket/bucket_mem_api.js b/tests/unit/bucket/bucket_mem_api.js index 980a5ba83f..8c0a2f3efd 100644 --- a/tests/unit/bucket/bucket_mem_api.js +++ b/tests/unit/bucket/bucket_mem_api.js @@ -12,30 +12,25 @@ const bucketName = 'Zaphod'; const objMD = { test: '8' }; const log = new DummyRequestLogger(); -describe('bucket API for getting, putting and deleting ' + - 'objects in a bucket', () => { +describe('bucket API for getting, putting and deleting ' + 'objects in a bucket', () => { let bucket; before(done => { cleanup(); const creationDate = new Date().toJSON(); - bucket = new BucketInfo(bucketName, 'iAmTheOwnerId', - 'iAmTheOwnerDisplayName', creationDate); + bucket = new BucketInfo(bucketName, 'iAmTheOwnerId', 'iAmTheOwnerDisplayName', creationDate); metadata.createBucket(bucketName, bucket, log, done); }); - it('should be able to add an object to a bucket ' + - 'and get the object by key', done => { + it('should be able to add an object to a bucket ' + 'and get the object by key', done => { metadata.putObjectMD(bucketName, 'sampleKey', objMD, {}, log, () => { - metadata.getObjectMD(bucketName, 'sampleKey', {}, log, - (err, value) => { + metadata.getObjectMD(bucketName, 'sampleKey', {}, log, (err, value) => { assert.deepStrictEqual(value, objMD); done(); }); }); }); - it('should return an error in response ' + - 'to getObjectMD when no such key', done => { + it('should return an error in response ' + 'to getObjectMD when no such key', done => { metadata.getObjectMD(bucketName, 'notThere', {}, log, (err, value) => { assert.strictEqual(err.is.NoSuchKey, true); assert.strictEqual(value, undefined); @@ -44,22 +39,18 @@ describe('bucket API for getting, putting and deleting ' + }); it('should be able to delete an object from a bucket', done => { - metadata.putObjectMD(bucketName, 'objectToDelete', '{}', {}, log, - () => { - metadata.deleteObjectMD(bucketName, 'objectToDelete', {}, log, - () => { - metadata.getObjectMD(bucketName, 'objectToDelete', {}, log, - (err, value) => { - assert.strictEqual(err.is.NoSuchKey, true); - assert.strictEqual(value, undefined); - done(); - }); + metadata.putObjectMD(bucketName, 'objectToDelete', '{}', {}, log, () => { + metadata.deleteObjectMD(bucketName, 'objectToDelete', {}, log, () => { + metadata.getObjectMD(bucketName, 'objectToDelete', {}, log, (err, value) => { + assert.strictEqual(err.is.NoSuchKey, true); + assert.strictEqual(value, undefined); + done(); + }); }); }); }); }); - describe('bucket API for getting a subset of objects from a bucket', () => { /* * Implementation of AWS GET Bucket (List Objects) functionality @@ -104,147 +95,127 @@ describe('bucket API for getting a subset of objects from a bucket', () => { before(done => { cleanup(); const creationDate = new Date().toJSON(); - bucket = new BucketInfo(bucketName, 'ownerid', - 'ownerdisplayname', creationDate); + bucket = new BucketInfo(bucketName, 'ownerid', 'ownerdisplayname', creationDate); metadata.createBucket(bucketName, bucket, log, done); }); - it('should return individual key if key does not contain ' + - 'the delimiter even if key contains prefix', done => { - async.waterfall([ - next => - metadata.putObjectMD(bucketName, 'key1', '{}', {}, log, next), - (data, next) => - metadata.putObjectMD(bucketName, 'noMatchKey', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'key1/', '{}', {}, log, next), - (data, next) => - metadata.listObject(bucketName, { prefix: 'key', delimiter, - maxKeys: defaultLimit }, log, next), - ], (err, response) => { - assert.strictEqual(isKeyInContents(response, 'key1'), true); - assert.strictEqual(response.CommonPrefixes.indexOf('key1'), -1); - assert.strictEqual(isKeyInContents(response, 'key1/'), false); - assert(response.CommonPrefixes.indexOf('key1/') > -1); - assert.strictEqual(isKeyInContents(response, 'noMatchKey'), false); - assert.strictEqual(response.CommonPrefixes.indexOf('noMatchKey'), - -1); - done(); - }); - }); - - it('should return grouped keys under common prefix if keys start with ' + - 'given prefix and contain given delimiter', done => { - async.waterfall([ - next => - metadata.putObjectMD(bucketName, 'key/one', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'key/two', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'key/three', '{}', {}, log, - next), - (data, next) => - metadata.listObject(bucketName, { prefix: 'ke', delimiter, - maxKeys: defaultLimit }, log, next), - ], (err, response) => { - assert(response.CommonPrefixes.indexOf('key/') > -1); - assert.strictEqual(isKeyInContents(response, 'key/'), false); - done(); - }); + it('should return individual key if key does not contain ' + 'the delimiter even if key contains prefix', done => { + async.waterfall( + [ + next => metadata.putObjectMD(bucketName, 'key1', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'noMatchKey', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'key1/', '{}', {}, log, next), + (data, next) => + metadata.listObject(bucketName, { prefix: 'key', delimiter, maxKeys: defaultLimit }, log, next), + ], + (err, response) => { + assert.strictEqual(isKeyInContents(response, 'key1'), true); + assert.strictEqual(response.CommonPrefixes.indexOf('key1'), -1); + assert.strictEqual(isKeyInContents(response, 'key1/'), false); + assert(response.CommonPrefixes.indexOf('key1/') > -1); + assert.strictEqual(isKeyInContents(response, 'noMatchKey'), false); + assert.strictEqual(response.CommonPrefixes.indexOf('noMatchKey'), -1); + done(); + }, + ); }); - it('should return grouped keys if no prefix ' + - 'given and keys match before delimiter', done => { + it( + 'should return grouped keys under common prefix if keys start with ' + + 'given prefix and contain given delimiter', + done => { + async.waterfall( + [ + next => metadata.putObjectMD(bucketName, 'key/one', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'key/two', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'key/three', '{}', {}, log, next), + (data, next) => + metadata.listObject(bucketName, { prefix: 'ke', delimiter, maxKeys: defaultLimit }, log, next), + ], + (err, response) => { + assert(response.CommonPrefixes.indexOf('key/') > -1); + assert.strictEqual(isKeyInContents(response, 'key/'), false); + done(); + }, + ); + }, + ); + + it('should return grouped keys if no prefix ' + 'given and keys match before delimiter', done => { metadata.putObjectMD(bucketName, 'noPrefix/one', '{}', {}, log, () => { - metadata.putObjectMD(bucketName, 'noPrefix/two', '{}', {}, log, - () => { - metadata.listObject(bucketName, { delimiter, - maxKeys: defaultLimit }, log, (err, response) => { - assert(response.CommonPrefixes.indexOf('noPrefix/') - > -1); - assert.strictEqual(isKeyInContents(response, - 'noPrefix'), false); - done(); - }); + metadata.putObjectMD(bucketName, 'noPrefix/two', '{}', {}, log, () => { + metadata.listObject(bucketName, { delimiter, maxKeys: defaultLimit }, log, (err, response) => { + assert(response.CommonPrefixes.indexOf('noPrefix/') > -1); + assert.strictEqual(isKeyInContents(response, 'noPrefix'), false); + done(); + }); }); }); }); - it('should return no grouped keys if no ' + - 'delimiter specified in getBucketListObjects', done => { - metadata.listObject(bucketName, - { prefix: 'key', maxKeys: defaultLimit }, log, - (err, response) => { - assert.strictEqual(response.CommonPrefixes.length, 0); - done(); - }); + it('should return no grouped keys if no ' + 'delimiter specified in getBucketListObjects', done => { + metadata.listObject(bucketName, { prefix: 'key', maxKeys: defaultLimit }, log, (err, response) => { + assert.strictEqual(response.CommonPrefixes.length, 0); + done(); + }); }); - it('should only return keys occurring alphabetically ' + - 'AFTER marker when no delimiter specified', done => { + it('should only return keys occurring alphabetically ' + 'AFTER marker when no delimiter specified', done => { metadata.putObjectMD(bucketName, 'a', '{}', {}, log, () => { metadata.putObjectMD(bucketName, 'b', '{}', {}, log, () => { - metadata.listObject(bucketName, - { marker: 'a', maxKeys: defaultLimit }, - log, (err, response) => { - assert(isKeyInContents(response, 'b')); - assert.strictEqual(isKeyInContents(response, 'a'), - false); - done(); - }); + metadata.listObject(bucketName, { marker: 'a', maxKeys: defaultLimit }, log, (err, response) => { + assert(isKeyInContents(response, 'b')); + assert.strictEqual(isKeyInContents(response, 'a'), false); + done(); + }); }); }); }); - it('should only return keys occurring alphabetically AFTER ' + - 'marker when delimiter specified', done => { - metadata.listObject(bucketName, - { marker: 'a', delimiter, maxKeys: defaultLimit }, - log, (err, response) => { - assert(isKeyInContents(response, 'b')); - assert.strictEqual(isKeyInContents(response, 'a'), false); - done(); - }); + it('should only return keys occurring alphabetically AFTER ' + 'marker when delimiter specified', done => { + metadata.listObject(bucketName, { marker: 'a', delimiter, maxKeys: defaultLimit }, log, (err, response) => { + assert(isKeyInContents(response, 'b')); + assert.strictEqual(isKeyInContents(response, 'a'), false); + done(); + }); }); - it('should only return keys occurring alphabetically AFTER ' + - 'marker when delimiter and prefix specified', done => { - metadata.listObject(bucketName, - { prefix: 'b', marker: 'a', delimiter, maxKeys: defaultLimit }, - log, (err, response) => { - assert(isKeyInContents(response, 'b')); - assert.strictEqual(isKeyInContents(response, 'a'), false); - done(); - }); - }); + it( + 'should only return keys occurring alphabetically AFTER ' + 'marker when delimiter and prefix specified', + done => { + metadata.listObject( + bucketName, + { prefix: 'b', marker: 'a', delimiter, maxKeys: defaultLimit }, + log, + (err, response) => { + assert(isKeyInContents(response, 'b')); + assert.strictEqual(isKeyInContents(response, 'a'), false); + done(); + }, + ); + }, + ); // Next marker should be the last common prefix or contents key returned it('should return a NextMarker if maxKeys reached', done => { - async.waterfall([ - next => - metadata.putObjectMD(bucketName, 'next/', '{}', {}, log, next), - (data, next) => - metadata.putObjectMD(bucketName, 'next/rollUp', '{}', {}, log, - next), - (data, next) => - metadata.putObjectMD(bucketName, 'next1/', '{}', {}, log, next), - (data, next) => - metadata.listObject(bucketName, - { prefix: 'next', delimiter, maxKeys: smallLimit }, - log, next), - ], (err, response) => { - assert(response.CommonPrefixes.indexOf('next/') > -1); - assert.strictEqual(response.CommonPrefixes.indexOf('next1/'), -1); - assert.strictEqual(response.NextMarker, 'next/'); - assert(response.IsTruncated); - done(); - }); + async.waterfall( + [ + next => metadata.putObjectMD(bucketName, 'next/', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'next/rollUp', '{}', {}, log, next), + (data, next) => metadata.putObjectMD(bucketName, 'next1/', '{}', {}, log, next), + (data, next) => + metadata.listObject(bucketName, { prefix: 'next', delimiter, maxKeys: smallLimit }, log, next), + ], + (err, response) => { + assert(response.CommonPrefixes.indexOf('next/') > -1); + assert.strictEqual(response.CommonPrefixes.indexOf('next1/'), -1); + assert.strictEqual(response.NextMarker, 'next/'); + assert(response.IsTruncated); + done(); + }, + ); }); }); - describe('stress test for bucket API', function describe() { this.timeout(200000); @@ -270,86 +241,83 @@ describe('stress test for bucket API', function describe() { before(done => { cleanup(); const creationDate = new Date().toJSON(); - bucket = new BucketInfo(bucketName, 'ownerid', - 'ownerdisplayname', creationDate); + bucket = new BucketInfo(bucketName, 'ownerid', 'ownerdisplayname', creationDate); metadata.createBucket(bucketName, bucket, log, done); }); - it(`should put ${numKeys} keys into bucket and retrieve bucket list ` + - `in under ${maxMilliseconds} milliseconds`, done => { - const data = {}; - const keys = []; - - // Create dictionary entries based on prefixes array - for (let i = 0; i < prefixes.length; i++) { - data[prefixes[i]] = []; - } - // Populate dictionary with random key extensions - let prefix; - for (let j = 0; j < numKeys; j++) { - prefix = prefixes[j % prefixes.length]; - data[prefix].push(makeid(10)); - } - - // Populate keys array with all keys including prefixes - Object.keys(data).forEach(dkey => { - data[dkey].forEach(key => { - keys.push(dkey + delimiter + key); + it( + `should put ${numKeys} keys into bucket and retrieve bucket list ` + `in under ${maxMilliseconds} milliseconds`, + done => { + const data = {}; + const keys = []; + + // Create dictionary entries based on prefixes array + for (let i = 0; i < prefixes.length; i++) { + data[prefixes[i]] = []; + } + // Populate dictionary with random key extensions + let prefix; + for (let j = 0; j < numKeys; j++) { + prefix = prefixes[j % prefixes.length]; + data[prefix].push(makeid(10)); + } + + // Populate keys array with all keys including prefixes + Object.keys(data).forEach(dkey => { + data[dkey].forEach(key => { + keys.push(dkey + delimiter + key); + }); }); - }); - // Shuffle the keys array so the keys appear in random order - shuffle(keys); + // Shuffle the keys array so the keys appear in random order + shuffle(keys); - // Start timing - const startTime = process.hrtime(); + // Start timing + const startTime = process.hrtime(); - async.each(keys, (item, next) => { - metadata.putObjectMD(bucketName, item, '{}', {}, log, next); - }, err => { - if (err) { - assert.strictEqual(err, undefined); - done(); - } else { - metadata.listObject(bucketName, { delimiter }, - log, (err, response) => { - // Stop timing and calculate millisecond time difference - const diff = timeDiff(startTime); - assert(diff < maxMilliseconds); - prefixes.forEach(prefix => { - assert(response.CommonPrefixes - .indexOf(prefix + delimiter) > -1); - }); + async.each( + keys, + (item, next) => { + metadata.putObjectMD(bucketName, item, '{}', {}, log, next); + }, + err => { + if (err) { + assert.strictEqual(err, undefined); done(); - }); - } + } else { + metadata.listObject(bucketName, { delimiter }, log, (err, response) => { + // Stop timing and calculate millisecond time difference + const diff = timeDiff(startTime); + assert(diff < maxMilliseconds); + prefixes.forEach(prefix => { + assert(response.CommonPrefixes.indexOf(prefix + delimiter) > -1); + }); + done(); + }); + } + }, + ); + }, + ); + + it('should return all keys as Contents if delimiter ' + 'does not match and specify NextMarker', done => { + metadata.listObject(bucketName, { delimiter: oddDelimiter, maxKeys: testLimit }, log, (err, response) => { + assert.strictEqual(response.CommonPrefixes.length, 0); + assert.strictEqual(response.Contents.length, testLimit); + assert.strictEqual(response.IsTruncated, true); + assert.strictEqual(typeof response.NextMarker, 'string'); + done(); }); }); - it('should return all keys as Contents if delimiter ' + - 'does not match and specify NextMarker', done => { - metadata.listObject(bucketName, - { delimiter: oddDelimiter, maxKeys: testLimit }, - log, (err, response) => { - assert.strictEqual(response.CommonPrefixes.length, 0); - assert.strictEqual(response.Contents.length, testLimit); - assert.strictEqual(response.IsTruncated, true); - assert.strictEqual(typeof response.NextMarker, 'string'); - done(); - }); - }); - - it('should return only keys occurring ' + - 'after specified marker', done => { - metadata.listObject(bucketName, { marker: testMarker, delimiter }, log, - (err, res) => { - assert.strictEqual(res.CommonPrefixes.length, - prefixes.length - 1); - assert.strictEqual(res.CommonPrefixes.indexOf(testPrefix), -1); - assert.strictEqual(res.Contents.length, 0); - assert.strictEqual(res.IsTruncated, false); - assert.strictEqual(res.NextMarker, undefined); - done(); - }); + it('should return only keys occurring ' + 'after specified marker', done => { + metadata.listObject(bucketName, { marker: testMarker, delimiter }, log, (err, res) => { + assert.strictEqual(res.CommonPrefixes.length, prefixes.length - 1); + assert.strictEqual(res.CommonPrefixes.indexOf(testPrefix), -1); + assert.strictEqual(res.Contents.length, 0); + assert.strictEqual(res.IsTruncated, false); + assert.strictEqual(res.NextMarker, undefined); + done(); + }); }); }); diff --git a/tests/unit/encryption/checkHealth.js b/tests/unit/encryption/checkHealth.js index e6b5d90c64..4bad434042 100644 --- a/tests/unit/encryption/checkHealth.js +++ b/tests/unit/encryption/checkHealth.js @@ -61,10 +61,12 @@ describe('KMS.checkHealth', () => { assert(shouldRefreshStub.calledOnce, 'shouldRefresh should be called once'); - assert(setResultSpy.calledOnceWithExactly({ - code: 200, - message: 'OK', - })); + assert( + setResultSpy.calledOnceWithExactly({ + code: 200, + message: 'OK', + }), + ); done(); }); @@ -89,11 +91,13 @@ describe('KMS.checkHealth', () => { assert(shouldRefreshStub.calledOnce, 'shouldRefresh should be called once'); - assert(setResultSpy.calledOnceWithExactly({ - code: 500, - message: 'KMS health check failed', - description: 'We encountered an internal error. Please try again.', - })); + assert( + setResultSpy.calledOnceWithExactly({ + code: 500, + message: 'KMS health check failed', + description: 'We encountered an internal error. Please try again.', + }), + ); done(); }); diff --git a/tests/unit/encryption/healthCheckCache.js b/tests/unit/encryption/healthCheckCache.js index 4d21232e8f..8cb66cfa49 100644 --- a/tests/unit/encryption/healthCheckCache.js +++ b/tests/unit/encryption/healthCheckCache.js @@ -68,7 +68,7 @@ describe('Cache Class', () => { it('should return false if elapsed time is less than duration minus maximum jitter', () => { const fakeNow = 1625077800000; - const fakeLastChecked = fakeNow - (45 * 60 * 1000); // 45 minutes ago + const fakeLastChecked = fakeNow - 45 * 60 * 1000; // 45 minutes ago sandbox.stub(Date, 'now').returns(fakeNow); sandbox.stub(Math, 'random').returns(0); cache.lastChecked = fakeLastChecked; @@ -80,7 +80,7 @@ describe('Cache Class', () => { it('should return true if elapsed time is greater than duration minus maximum jitter', () => { const fakeNow = 1625077800000; - const fakeLastChecked = fakeNow - (61 * 60 * 1000); // 61 minutes ago + const fakeLastChecked = fakeNow - 61 * 60 * 1000; // 61 minutes ago sandbox.stub(Date, 'now').returns(fakeNow); sandbox.stub(Math, 'random').returns(0); cache.lastChecked = fakeLastChecked; @@ -96,7 +96,7 @@ describe('Cache Class', () => { sandbox.stub(Date, 'now').returns(fakeNow); // Elapsed time = 5 hours - const fakeLastChecked1 = fakeNow - (5 * 60 * 60 * 1000); + const fakeLastChecked1 = fakeNow - 5 * 60 * 60 * 1000; cache.lastChecked = fakeLastChecked1; sandbox.stub(Math, 'random').returns(0); @@ -105,19 +105,15 @@ describe('Cache Class', () => { assert.strictEqual( cache.shouldRefresh(customDuration), false, - 'Cache should not refresh within custom duration' + 'Cache should not refresh within custom duration', ); // Elapsed time = 7 hours - const fakeLastChecked2 = fakeNow - (7 * 60 * 60 * 1000); + const fakeLastChecked2 = fakeNow - 7 * 60 * 60 * 1000; cache.lastChecked = fakeLastChecked2; // 7 hours > 6 hours => shouldRefresh = true - assert.strictEqual( - cache.shouldRefresh(customDuration), - true, - 'Cache should refresh after custom duration' - ); + assert.strictEqual(cache.shouldRefresh(customDuration), true, 'Cache should refresh after custom duration'); }); }); diff --git a/tests/unit/encryption/kms.js b/tests/unit/encryption/kms.js index ff94a77d8a..98ff7d4fd2 100644 --- a/tests/unit/encryption/kms.js +++ b/tests/unit/encryption/kms.js @@ -6,8 +6,7 @@ const Common = require('../../../lib/kms/common'); const { cleanup, DummyRequestLogger } = require('../helpers'); const log = new DummyRequestLogger(); -const dummyBucket = new BucketInfo( - 'dummyBucket', 'dummyOwnerId', 'Joe, John', new Date().toJSON()); +const dummyBucket = new BucketInfo('dummyBucket', 'dummyOwnerId', 'Joe, John', new Date().toJSON()); describe('KMS unit tests', () => { beforeEach(() => { @@ -20,17 +19,15 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption': algorithm, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - assert.strictEqual(sseInfo.cryptoScheme, 1); - assert.strictEqual(sseInfo.mandatory, true); - assert.strictEqual(sseInfo.algorithm, algorithm); - assert.notEqual(sseInfo.masterKeyId, undefined); - assert.notEqual(sseInfo.masterKeyId, null); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + assert.strictEqual(sseInfo.cryptoScheme, 1); + assert.strictEqual(sseInfo.mandatory, true); + assert.strictEqual(sseInfo.algorithm, algorithm); + assert.notEqual(sseInfo.masterKeyId, undefined); + assert.notEqual(sseInfo.masterKeyId, null); + done(); + }); }); it('should construct a sse info object on aws:kms', done => { @@ -41,49 +38,48 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption-aws-kms-key-id': masterKeyId, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - assert.strictEqual(sseInfo.cryptoScheme, 1); - assert.strictEqual(sseInfo.mandatory, true); - assert.strictEqual(sseInfo.algorithm, 'aws:kms'); - assert.strictEqual(sseInfo.configuredMasterKeyId, `${KMS.arnPrefix}${masterKeyId}`); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + assert.strictEqual(sseInfo.cryptoScheme, 1); + assert.strictEqual(sseInfo.mandatory, true); + assert.strictEqual(sseInfo.algorithm, 'aws:kms'); + assert.strictEqual(sseInfo.configuredMasterKeyId, `${KMS.arnPrefix}${masterKeyId}`); + done(); + }); }); - it('should not construct a sse info object if ' + - 'x-amz-scal-server-side-encryption header contains invalid ' + - 'algorithm option', done => { - const algorithm = 'garbage'; - const masterKeyId = 'foobarbaz'; - const headers = { - 'x-amz-scal-server-side-encryption': algorithm, - 'x-amz-scal-server-side-encryption-aws-kms-key-id': masterKeyId, - }; - const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { + it( + 'should not construct a sse info object if ' + + 'x-amz-scal-server-side-encryption header contains invalid ' + + 'algorithm option', + done => { + const algorithm = 'garbage'; + const masterKeyId = 'foobarbaz'; + const headers = { + 'x-amz-scal-server-side-encryption': algorithm, + 'x-amz-scal-server-side-encryption-aws-kms-key-id': masterKeyId, + }; + const sseConfig = parseBucketEncryptionHeaders(headers); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { assert.strictEqual(err, null); assert.strictEqual(sseInfo, null); done(); }); - }); + }, + ); - it('should not construct a sse info object if no ' + - 'x-amz-scal-server-side-encryption header included with request', + it( + 'should not construct a sse info object if no ' + + 'x-amz-scal-server-side-encryption header included with request', done => { const sseConfig = parseBucketEncryptionHeaders({}); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - assert.strictEqual(sseInfo, null); - done(); - }); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + assert.strictEqual(sseInfo, null); + done(); + }); + }, + ); it('should create a cipher bundle for AES256', done => { const algorithm = 'AES256'; @@ -91,22 +87,16 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption': algorithm, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - KMS.createCipherBundle( - sseInfo, log, (err, cipherBundle) => { - assert.strictEqual(cipherBundle.algorithm, - sseInfo.algorithm); - assert.strictEqual(cipherBundle.masterKeyId, - sseInfo.masterKeyId); - assert.strictEqual(cipherBundle.cryptoScheme, - sseInfo.cryptoScheme); - assert.notEqual(cipherBundle.cipheredDataKey, null); - assert.notEqual(cipherBundle.cipher, null); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + KMS.createCipherBundle(sseInfo, log, (err, cipherBundle) => { + assert.strictEqual(cipherBundle.algorithm, sseInfo.algorithm); + assert.strictEqual(cipherBundle.masterKeyId, sseInfo.masterKeyId); + assert.strictEqual(cipherBundle.cryptoScheme, sseInfo.cryptoScheme); + assert.notEqual(cipherBundle.cipheredDataKey, null); + assert.notEqual(cipherBundle.cipher, null); + done(); }); + }); }); it('should create a cipher bundle for aws:kms', done => { @@ -115,33 +105,24 @@ describe('KMS unit tests', () => { }; let masterKeyId; let sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - assert.strictEqual(err, null); - masterKeyId = sseInfo.bucketKeyId; - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + assert.strictEqual(err, null); + masterKeyId = sseInfo.bucketKeyId; + }); headers['x-amz-scal-server-side-encryption'] = 'aws:kms'; - headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] = - masterKeyId; + headers['x-amz-scal-server-side-encryption-aws-kms-key-id'] = masterKeyId; sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { - KMS.createCipherBundle( - sseInfo, log, (err, cipherBundle) => { - assert.strictEqual(cipherBundle.algorithm, - sseInfo.algorithm); - assert.strictEqual(cipherBundle.masterKeyId, - sseInfo.masterKeyId); - assert.strictEqual(cipherBundle.cryptoScheme, - sseInfo.cryptoScheme); - assert.notEqual(cipherBundle.cipheredDataKey, null); - assert.notEqual(cipherBundle.cipher, null); - done(); - }); + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + KMS.createCipherBundle(sseInfo, log, (err, cipherBundle) => { + assert.strictEqual(cipherBundle.algorithm, sseInfo.algorithm); + assert.strictEqual(cipherBundle.masterKeyId, sseInfo.masterKeyId); + assert.strictEqual(cipherBundle.cryptoScheme, sseInfo.cryptoScheme); + assert.notEqual(cipherBundle.cipheredDataKey, null); + assert.notEqual(cipherBundle.cipher, null); + done(); }); + }); }); /* cb(err, cipherBundle, decipherBundle) */ @@ -151,37 +132,30 @@ describe('KMS unit tests', () => { 'x-amz-scal-server-side-encryption': algorithm, }; const sseConfig = parseBucketEncryptionHeaders(headers); - KMS.bucketLevelEncryption( - dummyBucket, sseConfig, log, - (err, sseInfo) => { + KMS.bucketLevelEncryption(dummyBucket, sseConfig, log, (err, sseInfo) => { + if (err) { + cb(err); + return; + } + KMS.createCipherBundle(sseInfo, log, (err, cipherBundle) => { if (err) { cb(err); return; } - KMS.createCipherBundle( - sseInfo, log, (err, cipherBundle) => { - if (err) { - cb(err); - return; - } - const creatingSseInfo = sseInfo; - creatingSseInfo.cipheredDataKey = - Buffer.from(cipherBundle.cipheredDataKey, 'base64'); - KMS.createDecipherBundle( - sseInfo, 0, log, (err, decipherBundle) => { - if (err) { - cb(err); - return; - } - assert.strictEqual(typeof decipherBundle, - 'object'); - assert.strictEqual(decipherBundle.cryptoScheme, - cipherBundle.cryptoScheme); - assert.notEqual(decipherBundle.decipher, null); - cb(null, cipherBundle, decipherBundle); - }); - }); + const creatingSseInfo = sseInfo; + creatingSseInfo.cipheredDataKey = Buffer.from(cipherBundle.cipheredDataKey, 'base64'); + KMS.createDecipherBundle(sseInfo, 0, log, (err, decipherBundle) => { + if (err) { + cb(err); + return; + } + assert.strictEqual(typeof decipherBundle, 'object'); + assert.strictEqual(decipherBundle.cryptoScheme, cipherBundle.cryptoScheme); + assert.notEqual(decipherBundle.decipher, null); + cb(null, cipherBundle, decipherBundle); + }); }); + }); } it('should cipher and decipher a datastream', done => { @@ -199,8 +173,7 @@ describe('KMS unit tests', () => { }); }); - it('should increment the IV by modifying the last two positions of ' + - 'the buffer', () => { + it('should increment the IV by modifying the last two positions of ' + 'the buffer', () => { const derivedIV = Buffer.from('aaaaaaff', 'hex'); const counter = 6; const incrementedIV = Common._incrementIV(derivedIV, counter); @@ -208,8 +181,7 @@ describe('KMS unit tests', () => { assert.deepStrictEqual(incrementedIV, expected); }); - it('should increment the IV by incrementing the last position of the ' + - 'buffer', () => { + it('should increment the IV by incrementing the last position of the ' + 'buffer', () => { const derivedIV = Buffer.from('aaaaaaf0', 'hex'); const counter = 6; const incrementedIV = Common._incrementIV(derivedIV, counter); @@ -217,8 +189,7 @@ describe('KMS unit tests', () => { assert.deepStrictEqual(incrementedIV, expected); }); - it('should increment the IV by shifting each position in the ' + - 'buffer', () => { + it('should increment the IV by shifting each position in the ' + 'buffer', () => { const derivedIV = Buffer.from('ffffffff', 'hex'); const counter = 1; const incrementedIV = Common._incrementIV(derivedIV, counter); diff --git a/tests/unit/githubScripts/asyncMigrationScripts.js b/tests/unit/githubScripts/asyncMigrationScripts.js index 40ce384e13..aa2e65ad7f 100644 --- a/tests/unit/githubScripts/asyncMigrationScripts.js +++ b/tests/unit/githubScripts/asyncMigrationScripts.js @@ -72,12 +72,10 @@ describe('CI async migration scripts', () => { tempDirs.push(dir); fs.mkdirSync(path.join(dir, 'lib'), { recursive: true }); - fs.writeFileSync(path.join(dir, 'lib/newFile.js'), [ - 'function badStyle(param, cb) {', - ' return cb(null, param);', - '}', - '', - ].join('\n')); + fs.writeFileSync( + path.join(dir, 'lib/newFile.js'), + ['function badStyle(param, cb) {', ' return cb(null, param);', '}', ''].join('\n'), + ); run('git', ['add', 'lib/newFile.js'], dir); const result = runNodeScript(checkDiffScript, dir); diff --git a/tests/unit/healthchecks/clientCheck.js b/tests/unit/healthchecks/clientCheck.js index 2e514af8b4..f636516cf7 100644 --- a/tests/unit/healthchecks/clientCheck.js +++ b/tests/unit/healthchecks/clientCheck.js @@ -30,19 +30,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should succeed when all backends are healthy', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { code: 200, message: 'OK' }, - 'sproxyd-loc2': { code: 200, message: 'OK' }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { code: 200, message: 'OK' }, + 'sproxyd-loc2': { code: 200, message: 'OK' }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert.ifError(err); @@ -58,19 +66,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should fail when ALL backends of data client fail while metadata is healthy', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, - 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, + 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert(err); @@ -87,19 +103,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should succeed when ONE data location fails but another is healthy', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, - 'sproxyd-loc2': { code: 200, message: 'OK' }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, + 'sproxyd-loc2': { code: 200, message: 'OK' }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert.ifError(err); @@ -115,19 +139,27 @@ describe('clientCheck - failure detection logic', () => { }); it('should fail when ALL backends of multiple clients fail', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, - 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { error: errors.InternalError, code: 500 }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { error: errors.InternalError, code: 500 }, + 'sproxyd-loc2': { error: errors.InternalError, code: 500 }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { error: errors.InternalError, code: 500 }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert(err); @@ -145,15 +177,21 @@ describe('clientCheck - failure detection logic', () => { it('should succeed when client returns empty result', done => { dataStub.callsFake((log, cb) => cb(null, {})); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, { - vault: { code: 200, message: 'OK' }, - })); - kmsStub.callsFake((log, cb) => cb(null, { - kms: { code: 200, message: 'OK' }, - })); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => + cb(null, { + vault: { code: 200, message: 'OK' }, + }), + ); + kmsStub.callsFake((log, cb) => + cb(null, { + kms: { code: 200, message: 'OK' }, + }), + ); clientCheck(false, log, (err, result) => { assert.ifError(err); @@ -167,34 +205,44 @@ describe('clientCheck - failure detection logic', () => { }); describe('external backend error handling', () => { - it('should NOT fail on external backend errors during normal operation ' + - '(flightCheckOnStartUp=false)', done => { - dataStub.callsFake((log, cb) => cb(null, { - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, {})); - kmsStub.callsFake((log, cb) => cb(null, {})); + it( + 'should NOT fail on external backend errors during normal operation ' + '(flightCheckOnStartUp=false)', + done => { + dataStub.callsFake((log, cb) => + cb(null, { + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => cb(null, {})); + kmsStub.callsFake((log, cb) => cb(null, {})); - clientCheck(false, log, (err, result) => { - assert.ifError(err); - assert.deepStrictEqual(result, { - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - metadata: { code: 200, message: 'OK' }, + clientCheck(false, log, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, { + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + metadata: { code: 200, message: 'OK' }, + }); + done(); }); - done(); - }); - }); + }, + ); it('should fail on external backend errors during startup (flightCheckOnStartUp=true)', done => { - dataStub.callsFake((log, cb) => cb(null, { - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); + dataStub.callsFake((log, cb) => + cb(null, { + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); vaultStub.callsFake((log, cb) => cb(null, {})); kmsStub.callsFake((log, cb) => cb(null, {})); @@ -209,27 +257,34 @@ describe('clientCheck - failure detection logic', () => { }); }); - it('should succeed when external backend fails but internal backend is healthy ' + - '(flightCheckOnStartUp=false)', done => { - dataStub.callsFake((log, cb) => cb(null, { - 'sproxyd-loc1': { code: 200, message: 'OK' }, - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - })); - metadataStub.callsFake((log, cb) => cb(null, { - metadata: { code: 200, message: 'OK' }, - })); - vaultStub.callsFake((log, cb) => cb(null, {})); - kmsStub.callsFake((log, cb) => cb(null, {})); + it( + 'should succeed when external backend fails but internal backend is healthy ' + + '(flightCheckOnStartUp=false)', + done => { + dataStub.callsFake((log, cb) => + cb(null, { + 'sproxyd-loc1': { code: 200, message: 'OK' }, + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + }), + ); + metadataStub.callsFake((log, cb) => + cb(null, { + metadata: { code: 200, message: 'OK' }, + }), + ); + vaultStub.callsFake((log, cb) => cb(null, {})); + kmsStub.callsFake((log, cb) => cb(null, {})); - clientCheck(false, log, (err, result) => { - assert.ifError(err); - assert.deepStrictEqual(result, { - 'sproxyd-loc1': { code: 200, message: 'OK' }, - 's3-backend': { error: errors.InternalError, code: 500, external: true }, - metadata: { code: 200, message: 'OK' }, + clientCheck(false, log, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, { + 'sproxyd-loc1': { code: 200, message: 'OK' }, + 's3-backend': { error: errors.InternalError, code: 500, external: true }, + metadata: { code: 200, message: 'OK' }, + }); + done(); }); - done(); - }); - }); + }, + ); }); }); diff --git a/tests/unit/helpers.js b/tests/unit/helpers.js index a8f1f594c4..8075da8df5 100644 --- a/tests/unit/helpers.js +++ b/tests/unit/helpers.js @@ -34,11 +34,9 @@ const testsRangeOnEmptyFile = [ function makeid(size) { let text = ''; - const possible = - 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; + const possible = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'; for (let i = 0; i < size; i += 1) { - text += possible - .charAt(Math.floor(Math.random() * possible.length)); + text += possible.charAt(Math.floor(Math.random() * possible.length)); } return text; } @@ -62,16 +60,14 @@ function timeDiff(startTime) { const timeArray = process.hrtime(startTime); // timeArray[0] is whole seconds // timeArray[1] is remaining nanoseconds - const milliseconds = (timeArray[0] * 1000) + (timeArray[1] / 1e6); + const milliseconds = timeArray[0] * 1000 + timeArray[1] / 1e6; return milliseconds; } function makeAuthInfo(accessKey, userName) { const canIdMap = { - accessKey1: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' - + 'cd47ef2be', - accessKey2: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' - + 'cd47ef2bf', + accessKey1: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' + 'cd47ef2be', + accessKey2: '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7' + 'cd47ef2bf', lifecycleKey1: '0123456789abcdef/lifecycle', default: crypto.randomBytes(32).toString('hex'), }; @@ -139,32 +135,23 @@ class WebsiteConfig { }); } - xml.push(''); + xml.push(''); if (this.IndexDocument) { - xml.push('', - `${this.IndexDocument.Suffix}`, - ''); + xml.push('', `${this.IndexDocument.Suffix}`, ''); } if (this.ErrorDocument) { - xml.push('', - `${this.ErrorDocument.Key}`, - ''); + xml.push('', `${this.ErrorDocument.Key}`, ''); } if (this.RedirectAllRequestsTo) { xml.push(''); if (this.RedirectAllRequestsTo.HostName) { - xml.push('', - `${this.RedirectAllRequestsTo.HostName})`, - ''); + xml.push('', `${this.RedirectAllRequestsTo.HostName})`, ''); } if (this.RedirectAllRequestsTo.Protocol) { - xml.push('', - `${this.RedirectAllRequestsTo.Protocol})`, - ''); + xml.push('', `${this.RedirectAllRequestsTo.Protocol})`, ''); } xml.push(''); } @@ -193,8 +180,7 @@ class WebsiteConfig { } } -function createAlteredRequest(alteredItems, objToAlter, - baseOuterObj, baseInnerObj) { +function createAlteredRequest(alteredItems, objToAlter, baseOuterObj, baseInnerObj) { const alteredRequest = Object.assign({}, baseOuterObj); const alteredNestedObj = Object.assign({}, baseInnerObj); Object.keys(alteredItems).forEach(key => { @@ -205,8 +191,8 @@ function createAlteredRequest(alteredItems, objToAlter, } function cleanup() { - metadata.buckets = new Map; - metadata.keyMaps = new Map; + metadata.buckets = new Map(); + metadata.keyMaps = new Map(); // Set data store array back to empty array ds.length = 0; // Set data store key count back to 1 @@ -276,19 +262,22 @@ class DummyRequestLogger { class CorsConfigTester { constructor(params) { - this._cors = [{ - allowedMethods: ['PUT', 'POST', 'DELETE'], - allowedOrigins: ['http://www.example.com'], - allowedHeaders: ['*'], - maxAgeSeconds: 3000, - exposeHeaders: ['x-amz-server-side-encryption'], - }, { - id: 'testid', - allowedMethods: ['GET'], - allowedOrigins: ['*'], - allowedHeaders: ['*'], - maxAgeSeconds: 3000, - }]; + this._cors = [ + { + allowedMethods: ['PUT', 'POST', 'DELETE'], + allowedOrigins: ['http://www.example.com'], + allowedHeaders: ['*'], + maxAgeSeconds: 3000, + exposeHeaders: ['x-amz-server-side-encryption'], + }, + { + id: 'testid', + allowedMethods: ['GET'], + allowedOrigins: ['*'], + allowedHeaders: ['*'], + maxAgeSeconds: 3000, + }, + ]; if (params) { Object.keys(params).forEach(key => { @@ -306,14 +295,12 @@ class CorsConfigTester { xml.push(''); this._cors.forEach(rule => { xml.push(''); - ['allowedMethods', 'allowedOrigins', 'allowedHeaders', - 'exposeHeaders', 'maxAgeSeconds'] - .forEach(key => { + ['allowedMethods', 'allowedOrigins', 'allowedHeaders', 'exposeHeaders', 'maxAgeSeconds'].forEach(key => { if (rule[key] && Array.isArray(rule[key])) { - const element = key === 'maxAgeSeconds' ? - key.charAt(0).toUpperCase() + key.slice(1) : - key.charAt(0).toUpperCase() + - key.slice(1, -1); + const element = + key === 'maxAgeSeconds' + ? key.charAt(0).toUpperCase() + key.slice(1) + : key.charAt(0).toUpperCase() + key.slice(1, -1); rule[key].forEach(value => { xml.push(`<${element}>${value}`); }); @@ -323,8 +310,7 @@ class CorsConfigTester { xml.push(`${rule.id}`); } if (rule.maxAgeSeconds && !Array.isArray(rule.maxAgeSeconds)) { - xml.push(`${rule.maxAgeSeconds}` + - ''); + xml.push(`${rule.maxAgeSeconds}` + ''); } xml.push(''); }); @@ -344,8 +330,7 @@ class CorsConfigTester { }; if (method === 'PUT') { request.post = body || this.constructXml(); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } @@ -384,10 +369,11 @@ const versioningTestUtils = { query: { versioning: '' }, actionImplicitDenies: false, }; - const xml = '' + - `${status}` + - ''; + const xml = + '' + + `${status}` + + ''; request.post = xml; return request; }, @@ -414,8 +400,7 @@ class TaggingConfigTester { constructXml() { const xml = []; - xml.push('' + - ' '); + xml.push('' + ' '); Object.keys(this._tags).forEach(key => { const value = this._tags[key]; xml.push(`${key}${value}`); @@ -437,8 +422,7 @@ class TaggingConfigTester { }; if (method === 'PUT') { request.post = body || this.constructXml(); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } @@ -455,8 +439,7 @@ class TaggingConfigTester { }; if (method === 'PUT') { request.post = body || this.constructXml(); - request.headers['content-md5'] = crypto.createHash('md5') - .update(request.post, 'utf8').digest('base64'); + request.headers['content-md5'] = crypto.createHash('md5').update(request.post, 'utf8').digest('base64'); } return request; } @@ -499,16 +482,13 @@ class AccessControlPolicy { } }); } - xml.push('', ''); + xml.push('', ''); _pushChildren(this.Owner); xml.push('', ''); this.AccessControlList.forEach(grant => { xml.push('', ``); _pushChildren(grant.Grantee); - xml.push('', - `${grant.Permission}`, - ''); + xml.push('', `${grant.Permission}`, ''); }); xml.push('', ''); return xml.join(''); @@ -516,9 +496,16 @@ class AccessControlPolicy { } function createRequestContext(apiMethod, request) { - return new RequestContext(request.headers, - request.query, request.bucketName, request.objectKey, - '127.0.0.1', false, apiMethod, 's3'); + return new RequestContext( + request.headers, + request.query, + request.bucketName, + request.objectKey, + '127.0.0.1', + false, + apiMethod, + 's3', + ); } module.exports = { diff --git a/tests/unit/internal/routeVeeam.js b/tests/unit/internal/routeVeeam.js index 49f2d3ba4b..21c9f62e19 100644 --- a/tests/unit/internal/routeVeeam.js +++ b/tests/unit/internal/routeVeeam.js @@ -44,21 +44,19 @@ describe('RouteVeeam: checkBucketAndKey', () => { }); }); - [ - ['test', 'badObjectKey', null, 'GET', log], - ].forEach(test => { + [['test', 'badObjectKey', null, 'GET', log]].forEach(test => { it(`should return InvalidArgument for "${test[1]}" object name`, () => { assert.strictEqual(routeVeeam.checkBucketAndKey(...test).is.InvalidArgument, true); }); }); - [ - ['test', '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', { random: 'queryparam' }, 'GET', log], - ].forEach(test => { - it(`should return InvalidRequest for "${test[1]}" object name`, () => { - assert.strictEqual(routeVeeam.checkBucketAndKey(...test).is.InvalidRequest, true); - }); - }); + [['test', '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', { random: 'queryparam' }, 'GET', log]].forEach( + test => { + it(`should return InvalidRequest for "${test[1]}" object name`, () => { + assert.strictEqual(routeVeeam.checkBucketAndKey(...test).is.InvalidRequest, true); + }); + }, + ); [ ['test', '.system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml', null, 'GET', log], @@ -234,14 +232,19 @@ describe('RouteVeeam: routeVeeam', () => { url: '/bucket/veeam', }); req.method = 'PATCH'; - routeVeeam.routeVeeam('127.0.0.1', req, { - setHeader: () => {}, - writeHead: () => {}, - end: data => { - assert(data.includes('MethodNotAllowed')); - done(); + routeVeeam.routeVeeam( + '127.0.0.1', + req, + { + setHeader: () => {}, + writeHead: () => {}, + end: data => { + assert(data.includes('MethodNotAllowed')); + done(); + }, + headersSent: false, }, - headersSent: false, - }, log); + log, + ); }); }); diff --git a/tests/unit/internal/veeam/schemas/system.js b/tests/unit/internal/veeam/schemas/system.js index ee92bfbf4c..06fa21ea9e 100644 --- a/tests/unit/internal/veeam/schemas/system.js +++ b/tests/unit/internal/veeam/schemas/system.js @@ -18,7 +18,7 @@ describe('RouteVeeam: validateSystemSchema 1.0', () => { CapacityInfo: true, UploadSessions: true, IAMSTS: true, - } + }, }, SystemRecommendations: { S3ConcurrentTaskLimit: 0, @@ -89,7 +89,6 @@ describe('RouteVeeam: validateSystemSchema 1.0', () => { }); }); - describe('RouteVeeam: validateSystemSchema unknown version', () => { const protocolVersion = '"1.1"'; [ @@ -101,7 +100,7 @@ describe('RouteVeeam: validateSystemSchema unknown version', () => { CapacityInfo: true, UploadSessions: true, IAMSTS: true, - } + }, }, SystemRecommendations: { S3ConcurrentTaskLimit: 0, diff --git a/tests/unit/management/agent.js b/tests/unit/management/agent.js index da806a7521..57c7c7c1a4 100644 --- a/tests/unit/management/agent.js +++ b/tests/unit/management/agent.js @@ -1,8 +1,6 @@ const assert = require('assert'); -const { - createWSAgent, -} = require('../../../lib/management/push'); +const { createWSAgent } = require('../../../lib/management/push'); const proxy = 'http://proxy:3128/'; const logger = { info: () => {} }; @@ -10,57 +8,93 @@ const logger = { info: () => {} }; function testVariableSet(httpProxy, httpsProxy, allProxy, noProxy) { return () => { it(`should use ${httpProxy} environment variable`, () => { - let agent = createWSAgent('https://pushserver', { - [httpProxy]: 'http://proxy:3128', - }, logger); + let agent = createWSAgent( + 'https://pushserver', + { + [httpProxy]: 'http://proxy:3128', + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('http://pushserver', { - [httpProxy]: proxy, - }, logger); + agent = createWSAgent( + 'http://pushserver', + { + [httpProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); it(`should use ${httpsProxy} environment variable`, () => { - let agent = createWSAgent('http://pushserver', { - [httpsProxy]: proxy, - }, logger); + let agent = createWSAgent( + 'http://pushserver', + { + [httpsProxy]: proxy, + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('https://pushserver', { - [httpsProxy]: proxy, - }, logger); + agent = createWSAgent( + 'https://pushserver', + { + [httpsProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); it(`should use ${allProxy} environment variable`, () => { - let agent = createWSAgent('http://pushserver', { - [allProxy]: proxy, - }, logger); + let agent = createWSAgent( + 'http://pushserver', + { + [allProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); - agent = createWSAgent('https://pushserver', { - [allProxy]: proxy, - }, logger); + agent = createWSAgent( + 'https://pushserver', + { + [allProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); it(`should use ${noProxy} environment variable`, () => { - let agent = createWSAgent('http://pushserver', { - [noProxy]: 'pushserver', - }, logger); + let agent = createWSAgent( + 'http://pushserver', + { + [noProxy]: 'pushserver', + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('http://pushserver', { - [noProxy]: 'pushserver', - [httpProxy]: proxy, - }, logger); + agent = createWSAgent( + 'http://pushserver', + { + [noProxy]: 'pushserver', + [httpProxy]: proxy, + }, + logger, + ); assert.equal(agent, null); - agent = createWSAgent('http://pushserver', { - [noProxy]: 'pushserver2', - [httpProxy]: proxy, - }, logger); + agent = createWSAgent( + 'http://pushserver', + { + [noProxy]: 'pushserver2', + [httpProxy]: proxy, + }, + logger, + ); assert.equal(agent.proxy.href, proxy); }); }; @@ -74,9 +108,7 @@ describe('Websocket connection agent', () => { }); }); - describe('with lowercase proxy env', - testVariableSet('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')); + describe('with lowercase proxy env', testVariableSet('http_proxy', 'https_proxy', 'all_proxy', 'no_proxy')); - describe('with uppercase proxy env', - testVariableSet('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY')); + describe('with uppercase proxy env', testVariableSet('HTTP_PROXY', 'HTTPS_PROXY', 'ALL_PROXY', 'NO_PROXY')); }); diff --git a/tests/unit/management/configuration.js b/tests/unit/management/configuration.js index 71134c4f3c..ef38fcd435 100644 --- a/tests/unit/management/configuration.js +++ b/tests/unit/management/configuration.js @@ -8,26 +8,20 @@ const metadata = require('../../../lib/metadata/wrapper'); const managementDatabaseName = 'PENSIEVE'; const tokenConfigurationKey = 'auth/zenko/remote-management-token'; -const { privateKey, accessKey, decryptedSecretKey, secretKey, canonicalId, - userName } = require('./resources.json'); +const { privateKey, accessKey, decryptedSecretKey, secretKey, canonicalId, userName } = require('./resources.json'); const shortid = '123456789012'; const email = 'customaccount1@setbyenv.com'; const arn = 'arn:aws:iam::123456789012:root'; const { config } = require('../../../lib/Config'); -const { - remoteOverlayIsNewer, - patchConfiguration, -} = require('../../../lib/management/configuration'); +const { remoteOverlayIsNewer, patchConfiguration } = require('../../../lib/management/configuration'); -const { - initManagementDatabase, -} = require('../../../lib/management/index'); +const { initManagementDatabase } = require('../../../lib/management/index'); function initManagementCredentialsMock(cb) { - return metadata.putObjectMD(managementDatabaseName, - tokenConfigurationKey, { privateKey }, {}, - log, error => cb(error)); + return metadata.putObjectMD(managementDatabaseName, tokenConfigurationKey, { privateKey }, {}, log, error => + cb(error), + ); } function getConfig() { @@ -37,14 +31,11 @@ function getConfig() { // Original Config const overlayVersionOriginal = Object.assign({}, config.overlayVersion); const authDataOriginal = Object.assign({}, config.authData).accounts; -const locationConstraintsOriginal = Object.assign({}, - config.locationConstraints); +const locationConstraintsOriginal = Object.assign({}, config.locationConstraints); const restEndpointsOriginal = Object.assign({}, config.restEndpoints); const browserAccessEnabledOriginal = config.browserAccessEnabled; const instanceId = '19683e55-56f7-4a4c-98a7-706c07e4ec30'; -const publicInstanceId = crypto.createHash('sha256') - .update(instanceId) - .digest('hex'); +const publicInstanceId = crypto.createHash('sha256').update(instanceId).digest('hex'); function resetConfig() { config.overlayVersion = overlayVersionOriginal; @@ -63,12 +54,14 @@ function assertConfig(actualConf, expectedConf) { } describe('patchConfiguration', () => { - before(done => initManagementDatabase(log, err => { - if (err) { - return done(err); - } - return initManagementCredentialsMock(done); - })); + before(done => + initManagementDatabase(log, err => { + if (err) { + return done(err); + } + return initManagementCredentialsMock(done); + }), + ); beforeEach(() => { resetConfig(); }); @@ -115,17 +108,21 @@ describe('patchConfiguration', () => { publicInstanceId, browserAccessEnabled: true, authData: { - accounts: [{ - name: userName, - email, - arn, - canonicalID: canonicalId, - shortid, - keys: [{ - access: accessKey, - secret: decryptedSecretKey, - }], - }], + accounts: [ + { + name: userName, + email, + arn, + canonicalID: canonicalId, + shortid, + keys: [ + { + access: accessKey, + secret: decryptedSecretKey, + }, + ], + }, + ], }, locationConstraints: { 'us-east-1': { @@ -141,14 +138,12 @@ describe('patchConfiguration', () => { }, }; assertConfig(actualConf, expectedConf); - assert.deepStrictEqual(actualConf.restEndpoints['1.1.1.1'], - 'us-east-1'); + assert.deepStrictEqual(actualConf.restEndpoints['1.1.1.1'], 'us-east-1'); return done(); }); }); - it('should apply second configuration if version (2) is greater than ' + - 'overlayVersion (1)', done => { + it('should apply second configuration if version (2) is greater than ' + 'overlayVersion (1)', done => { const newConf1 = { version: 1, instanceId, @@ -175,8 +170,7 @@ describe('patchConfiguration', () => { }); }); - it('should not apply the second configuration if version equals ' + - 'overlayVersion', done => { + it('should not apply the second configuration if version equals ' + 'overlayVersion', done => { const newConf1 = { version: 1, instanceId, @@ -205,40 +199,37 @@ describe('patchConfiguration', () => { }); describe('remoteOverlayIsNewer', () => { - it('should return remoteOverlayIsNewer equals false if remote overlay ' + - 'is less than the cached', () => { + it('should return remoteOverlayIsNewer equals false if remote overlay ' + 'is less than the cached', () => { const cachedOverlay = { version: 2, }; const remoteOverlay = { version: 1, }; - const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, - remoteOverlay); + const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, remoteOverlay); assert.equal(isRemoteOverlayNewer, false); }); - it('should return remoteOverlayIsNewer equals false if remote overlay ' + - 'and the cached one are equal', () => { + it('should return remoteOverlayIsNewer equals false if remote overlay ' + 'and the cached one are equal', () => { const cachedOverlay = { version: 1, }; const remoteOverlay = { version: 1, }; - const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, - remoteOverlay); + const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, remoteOverlay); assert.equal(isRemoteOverlayNewer, false); }); - it('should return remoteOverlayIsNewer equals true if remote overlay ' + - 'version is greater than the cached one ', () => { - const cachedOverlay = { - version: 0, - }; - const remoteOverlay = { - version: 1, - }; - const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, - remoteOverlay); - assert.equal(isRemoteOverlayNewer, true); - }); + it( + 'should return remoteOverlayIsNewer equals true if remote overlay ' + 'version is greater than the cached one ', + () => { + const cachedOverlay = { + version: 0, + }; + const remoteOverlay = { + version: 1, + }; + const isRemoteOverlayNewer = remoteOverlayIsNewer(cachedOverlay, remoteOverlay); + assert.equal(isRemoteOverlayNewer, true); + }, + ); }); diff --git a/tests/unit/management/secureChannel.js b/tests/unit/management/secureChannel.js index dc29bb1b8a..0f63441334 100644 --- a/tests/unit/management/secureChannel.js +++ b/tests/unit/management/secureChannel.js @@ -19,16 +19,16 @@ describe('report handler', () => { }); [ - { value: 'true', result: true }, - { value: 'TRUE', result: true }, - { value: 'tRuE', result: true }, - { value: '1', result: true }, - { value: 'false', result: false }, - { value: 'FALSE', result: false }, - { value: 'FaLsE', result: false }, - { value: '0', result: false }, - { value: 'foo', result: false }, - { value: '', result: true }, + { value: 'true', result: true }, + { value: 'TRUE', result: true }, + { value: 'tRuE', result: true }, + { value: '1', result: true }, + { value: 'false', result: false }, + { value: 'FALSE', result: false }, + { value: 'FaLsE', result: false }, + { value: '0', result: false }, + { value: 'foo', result: false }, + { value: '', result: true }, { value: undefined, result: true }, ].forEach(param => it(`should allow set local file system capability ${param.value}`, () => { @@ -40,6 +40,6 @@ describe('report handler', () => { assert.strictEqual(getCapabilities().locationTypeLocal, param.result); process.env = OLD_ENV; - }) + }), ); }); diff --git a/tests/unit/management/testChannelMessageV0.js b/tests/unit/management/testChannelMessageV0.js index 46a4bb2d2d..f43d6f7687 100644 --- a/tests/unit/management/testChannelMessageV0.js +++ b/tests/unit/management/testChannelMessageV0.js @@ -1,10 +1,6 @@ const assert = require('assert'); -const { - ChannelMessageV0, - MessageType, - TargetType, -} = require('../../../lib/management/ChannelMessageV0'); +const { ChannelMessageV0, MessageType, TargetType } = require('../../../lib/management/ChannelMessageV0'); const { CONFIG_OVERLAY_MESSAGE, diff --git a/tests/unit/multipleBackend/VersioningBackendClient.js b/tests/unit/multipleBackend/VersioningBackendClient.js index 60d8527602..065f34e08f 100644 --- a/tests/unit/multipleBackend/VersioningBackendClient.js +++ b/tests/unit/multipleBackend/VersioningBackendClient.js @@ -7,8 +7,7 @@ const DummyService = require('../DummyService'); const { DummyRequestLogger } = require('../helpers'); const missingVerIdInternalError = errorInstances.InternalError.customizeDescription( - 'Invalid state. Please ensure versioning is enabled ' + - 'in AWS for the location constraint and try again.' + 'Invalid state. Please ensure versioning is enabled ' + 'in AWS for the location constraint and try again.', ); const log = new DummyRequestLogger(); @@ -43,8 +42,7 @@ const s3Config = { }; const assertSuccess = (err, cb) => { - assert.ifError(err, - `Expected success, but got error ${err}`); + assert.ifError(err, `Expected success, but got error ${err}`); cb(); }; @@ -54,26 +52,22 @@ const assertFailure = (err, cb) => { }; const genTests = [ { - msg: 'should return success if supportsVersioning === true ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === true ' + 'and backend versioning is enabled', input: { supportsVersioning: true, enableMockVersioning: true }, callback: assertSuccess, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is enabled', input: { supportsVersioning: false, enableMockVersioning: true }, callback: assertSuccess, }, { - msg: 'should return error if supportsVersioning === true ' + - 'and backend versioning is disabled', + msg: 'should return error if supportsVersioning === true ' + 'and backend versioning is disabled', input: { supportsVersioning: true, enableMockVersioning: false }, callback: assertFailure, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is disabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is disabled', input: { supportsVersioning: false, enableMockVersioning: false }, callback: assertSuccess, }, @@ -86,12 +80,13 @@ describe('AwsClient::putObject', () => { testClient = new AwsClient(s3Config); testClient._client = new DummyService({ versioning: true }); }); - genTests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - testClient.put('', 0, { bucketName: bucket, objectKey: key }, - reqUID, err => test.callback(err, done)); - })); + genTests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.put('', 0, { bucketName: bucket, objectKey: key }, reqUID, err => test.callback(err, done)); + }), + ); }); describe('AwsClient::copyObject', () => { @@ -102,13 +97,15 @@ describe('AwsClient::copyObject', () => { testClient._client = new DummyService({ versioning: true }); }); - genTests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - testClient.copyObject(copyObjectRequest, null, key, - sourceLocationConstraint, null, config, log, - err => test.callback(err, done)); - })); + genTests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.copyObject(copyObjectRequest, null, key, sourceLocationConstraint, null, config, log, err => + test.callback(err, done), + ); + }), + ); }); describe('AwsClient::completeMPU', () => { @@ -118,13 +115,14 @@ describe('AwsClient::completeMPU', () => { testClient = new AwsClient(s3Config); testClient._client = new DummyService({ versioning: true }); }); - genTests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - const uploadId = 'externalBackendTestUploadId'; - testClient.completeMPU(jsonList, null, key, uploadId, - bucket, log, err => test.callback(err, done)); - })); + genTests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + const uploadId = 'externalBackendTestUploadId'; + testClient.completeMPU(jsonList, null, key, uploadId, bucket, log, err => test.callback(err, done)); + }), + ); }); describe('AwsClient::healthcheck', () => { @@ -159,34 +157,31 @@ describe('AwsClient::healthcheck', () => { const tests = [ { - msg: 'should return success if supportsVersioning === true ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === true ' + 'and backend versioning is enabled', input: { supportsVersioning: true, enableMockVersioning: true }, callback: assertSuccessVersioned, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is enabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is enabled', input: { supportsVersioning: false, enableMockVersioning: true }, callback: assertSuccessNonVersioned, }, { - msg: 'should return error if supportsVersioning === true ' + - ' and backend versioning is disabled', + msg: 'should return error if supportsVersioning === true ' + ' and backend versioning is disabled', input: { supportsVersioning: true, enableMockVersioning: false }, callback: assertFailure, }, { - msg: 'should return success if supportsVersioning === false ' + - 'and backend versioning is disabled', + msg: 'should return success if supportsVersioning === false ' + 'and backend versioning is disabled', input: { supportsVersioning: false, enableMockVersioning: false }, callback: assertSuccessNonVersioned, }, ]; - tests.forEach(test => it(test.msg, done => { - testClient._supportsVersioning = test.input.supportsVersioning; - testClient._client.versioning = test.input.enableMockVersioning; - testClient.healthcheck('backend', - (err, resp) => test.callback(resp.backend, done)); - })); + tests.forEach(test => + it(test.msg, done => { + testClient._supportsVersioning = test.input.supportsVersioning; + testClient._client.versioning = test.input.enableMockVersioning; + testClient.healthcheck('backend', (err, resp) => test.callback(resp.backend, done)); + }), + ); }); diff --git a/tests/unit/multipleBackend/getReplicationBackendDataLocator.js b/tests/unit/multipleBackend/getReplicationBackendDataLocator.js index b27f7595ff..048928725a 100644 --- a/tests/unit/multipleBackend/getReplicationBackendDataLocator.js +++ b/tests/unit/multipleBackend/getReplicationBackendDataLocator.js @@ -1,7 +1,6 @@ const assert = require('assert'); -const getReplicationBackendDataLocator = require( - '../../../lib/api/apiUtils/object/getReplicationBackendDataLocator'); +const getReplicationBackendDataLocator = require('../../../lib/api/apiUtils/object/getReplicationBackendDataLocator'); const locCheckResult = { location: 'spoofbackend', @@ -9,47 +8,45 @@ const locCheckResult = { locationType: 'spoof', }; const repNoMatch = { backends: [{ site: 'nomatch' }] }; -const repMatchPending = { backends: - [{ site: 'spoofbackend', status: 'PENDING', dataVersionId: '' }] }; -const repMatchFailed = { backends: - [{ site: 'spoofbackend', status: 'FAILED', dataVersionId: '' }] }; -const repMatch = { backends: [{ - site: 'spoofbackend', - status: 'COMPLETED', - dataStoreVersionId: 'spoofid' }], +const repMatchPending = { backends: [{ site: 'spoofbackend', status: 'PENDING', dataVersionId: '' }] }; +const repMatchFailed = { backends: [{ site: 'spoofbackend', status: 'FAILED', dataVersionId: '' }] }; +const repMatch = { + backends: [ + { + site: 'spoofbackend', + status: 'COMPLETED', + dataStoreVersionId: 'spoofid', + }, + ], }; -const expDataLocator = [{ - key: locCheckResult.key, - dataStoreName: locCheckResult.location, - dataStoreType: locCheckResult.locationType, - dataStoreVersionId: repMatch.backends[0].dataStoreVersionId, -}]; - +const expDataLocator = [ + { + key: locCheckResult.key, + dataStoreName: locCheckResult.location, + dataStoreType: locCheckResult.locationType, + dataStoreVersionId: repMatch.backends[0].dataStoreVersionId, + }, +]; describe('Replication Backend Compare', () => { it('should return error if no match in replication backends', () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repNoMatch); + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repNoMatch); assert.strictEqual(repBackendResult.error.is.InvalidLocationConstraint, true); }); it('should return a status and reason if backend status is PENDING', () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repMatchPending); + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repMatchPending); assert.strictEqual(repBackendResult.dataLocator, undefined); assert.strictEqual(repBackendResult.status, 'PENDING'); assert.notStrictEqual(repBackendResult.reason, undefined); }); it('should return a status and reason if backend status is FAILED', () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repMatchFailed); + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repMatchFailed); assert.strictEqual(repBackendResult.dataLocator, undefined); assert.strictEqual(repBackendResult.status, 'FAILED'); assert.notStrictEqual(repBackendResult.reason, undefined); }); - it('should return dataLocator obj if backend matches and rep is COMPLETED', - () => { - const repBackendResult = - getReplicationBackendDataLocator(locCheckResult, repMatch); + it('should return dataLocator obj if backend matches and rep is COMPLETED', () => { + const repBackendResult = getReplicationBackendDataLocator(locCheckResult, repMatch); assert.strictEqual(repBackendResult.status, 'COMPLETED'); assert.deepStrictEqual(repBackendResult.dataLocator, expDataLocator); }); diff --git a/tests/unit/multipleBackend/locationConstraintCheck.js b/tests/unit/multipleBackend/locationConstraintCheck.js index 1388e53bbe..dbba7c3fe0 100644 --- a/tests/unit/multipleBackend/locationConstraintCheck.js +++ b/tests/unit/multipleBackend/locationConstraintCheck.js @@ -3,8 +3,7 @@ const assert = require('assert'); const { BucketInfo, BackendInfo } = require('arsenal').models; const DummyRequest = require('../DummyRequest'); const { DummyRequestLogger } = require('../helpers'); -const locationConstraintCheck - = require('../../../lib/api/apiUtils/object/locationConstraintCheck'); +const locationConstraintCheck = require('../../../lib/api/apiUtils/object/locationConstraintCheck'); const memLocation = 'scality-internal-mem'; const fileLocation = 'scality-internal-file'; @@ -18,48 +17,51 @@ const objectKey = 'someobject'; const postBody = Buffer.from('I am a body', 'utf8'); const log = new DummyRequestLogger(); -const testBucket = new BucketInfo(bucketName, owner, ownerDisplayName, - testDate, null, null, null, null, null, null, locationConstraint); +const testBucket = new BucketInfo( + bucketName, + owner, + ownerDisplayName, + testDate, + null, + null, + null, + null, + null, + null, + locationConstraint, +); function createTestRequest(locationConstraint) { - const testRequest = new DummyRequest({ - bucketName, - namespace, - objectKey, - headers: { 'x-amz-meta-scal-location-constraint': locationConstraint }, - url: `/${bucketName}/${objectKey}`, - parsedHost: 'localhost', - }, postBody); + const testRequest = new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { 'x-amz-meta-scal-location-constraint': locationConstraint }, + url: `/${bucketName}/${objectKey}`, + parsedHost: 'localhost', + }, + postBody, + ); return testRequest; } describe('Location Constraint Check', () => { - it('should return error if controlling location constraint is ' + - 'not valid', done => { - const backendInfoObj = locationConstraintCheck( - createTestRequest('fail-region'), null, testBucket, log); - assert.strictEqual(backendInfoObj.err.code, 400, - 'Expected "Invalid Argument" code error'); - assert(backendInfoObj.err.is.InvalidArgument, 'Expected "Invalid ' + - 'Argument" error'); + it('should return error if controlling location constraint is ' + 'not valid', done => { + const backendInfoObj = locationConstraintCheck(createTestRequest('fail-region'), null, testBucket, log); + assert.strictEqual(backendInfoObj.err.code, 400, 'Expected "Invalid Argument" code error'); + assert(backendInfoObj.err.is.InvalidArgument, 'Expected "Invalid ' + 'Argument" error'); done(); }); - it('should return instance of BackendInfo with correct ' + - 'locationConstraints', done => { - const backendInfoObj = locationConstraintCheck( - createTestRequest(memLocation), null, testBucket, log); - assert.strictEqual(backendInfoObj.err, null, 'Expected success ' + - `but got error ${backendInfoObj.err}`); + it('should return instance of BackendInfo with correct ' + 'locationConstraints', done => { + const backendInfoObj = locationConstraintCheck(createTestRequest(memLocation), null, testBucket, log); + assert.strictEqual(backendInfoObj.err, null, 'Expected success ' + `but got error ${backendInfoObj.err}`); assert.strictEqual(typeof backendInfoObj.controllingLC, 'string'); - assert.equal(backendInfoObj.backendInfo instanceof BackendInfo, - true); - assert.strictEqual(backendInfoObj. - backendInfo.getObjectLocationConstraint(), memLocation); - assert.strictEqual(backendInfoObj. - backendInfo.getBucketLocationConstraint(), fileLocation); - assert.strictEqual(backendInfoObj.backendInfo.getRequestEndpoint(), - 'localhost'); + assert.equal(backendInfoObj.backendInfo instanceof BackendInfo, true); + assert.strictEqual(backendInfoObj.backendInfo.getObjectLocationConstraint(), memLocation); + assert.strictEqual(backendInfoObj.backendInfo.getBucketLocationConstraint(), fileLocation); + assert.strictEqual(backendInfoObj.backendInfo.getRequestEndpoint(), 'localhost'); done(); }); }); diff --git a/tests/unit/multipleBackend/locationHeaderCheck.js b/tests/unit/multipleBackend/locationHeaderCheck.js index 34d58e0a9b..8a806ee351 100644 --- a/tests/unit/multipleBackend/locationHeaderCheck.js +++ b/tests/unit/multipleBackend/locationHeaderCheck.js @@ -1,8 +1,7 @@ const assert = require('assert'); const { errorInstances } = require('arsenal'); -const locationHeaderCheck = - require('../../../lib/api/apiUtils/object/locationHeaderCheck'); +const locationHeaderCheck = require('../../../lib/api/apiUtils/object/locationHeaderCheck'); const objectKey = 'locationHeaderCheckObject'; const bucketName = 'locationHeaderCheckBucket'; @@ -11,18 +10,22 @@ const testCases = [ { location: 'doesnotexist', expRes: errorInstances.InvalidLocationConstraint.customizeDescription( - 'Invalid location constraint specified in header'), - }, { + 'Invalid location constraint specified in header', + ), + }, + { location: '', expRes: undefined, - }, { + }, + { location: 'awsbackend', expRes: { location: 'awsbackend', key: objectKey, locationType: 'aws_s3', }, - }, { + }, + { location: 'awsbackendmismatch', expRes: { location: 'awsbackendmismatch', @@ -34,11 +37,9 @@ const testCases = [ describe('Location Header Check', () => { testCases.forEach(test => { - it('should return expected result with location constraint header ' + - `set to ${test.location}`, () => { + it('should return expected result with location constraint header ' + `set to ${test.location}`, () => { const headers = { 'x-amz-location-constraint': `${test.location}` }; - const checkRes = - locationHeaderCheck(headers, objectKey, bucketName); + const checkRes = locationHeaderCheck(headers, objectKey, bucketName); assert.deepStrictEqual(checkRes, test.expRes); }); }); diff --git a/tests/unit/policies.js b/tests/unit/policies.js index b1dfdc596e..5e2dda6a23 100644 --- a/tests/unit/policies.js +++ b/tests/unit/policies.js @@ -240,10 +240,7 @@ const apiMatrix = [ headers: { 'x-amz-version-id': '1', }, - expectedPermissions: [ - 's3:PutObject', - 's3:PutObjectVersionTagging', - ], + expectedPermissions: ['s3:PutObject', 's3:PutObjectVersionTagging'], }, { name: 'objectPutACL', @@ -282,7 +279,6 @@ const apiMatrix = [ }, ]; - function prepareDummyRequest(headers = {}) { const request = new DummyRequest({ hostname: 'localhost', @@ -300,13 +296,16 @@ describe('Policies: permission checks for S3 APIs', () => { if (api.name.length === 0) { return; } - const message = `should return ${api.expectedPermissions.join(', ')} in requestContextParams for ${api.name}` + - `${(api.headers && api.headers.length) > 0 ? - ` with headers ${api.headers.map(el => el[0]).join(', ')}` : ''}`; + const message = + `should return ${api.expectedPermissions.join(', ')} in requestContextParams for ${api.name}` + + `${ + (api.headers && api.headers.length) > 0 + ? ` with headers ${api.headers.map(el => el[0]).join(', ')}` + : '' + }`; it(message, () => { const request = prepareDummyRequest(api.headers); - const requestContexts = prepareRequestContexts(api.name, request, - sourceBucket, sourceObject); + const requestContexts = prepareRequestContexts(api.name, request, sourceBucket, sourceObject); const requestedActions = requestContexts.map(rq => rq.getAction()); assert.deepStrictEqual(requestedActions, api.expectedPermissions); }); @@ -320,26 +319,26 @@ describe('Policies: permission checks for S3 APIs', () => { } it('should return s3:PutBucket without any provided header', () => { - assert.deepStrictEqual( - putBucketApiMethods(), - ['bucketPut'], - ); + assert.deepStrictEqual(putBucketApiMethods(), ['bucketPut']); }); - it('should return s3:CreateBucket, s3:PutBucketVersioning and s3:PutBucketObjectLockConfiguration' + - ' with object-lock headers', () => { - assert.deepStrictEqual( - putBucketApiMethods({ 'x-amz-bucket-object-lock-enabled': 'true' }), - ['bucketPut', 'bucketPutObjectLock', 'bucketPutVersioning'], - ); - }); + it( + 'should return s3:CreateBucket, s3:PutBucketVersioning and s3:PutBucketObjectLockConfiguration' + + ' with object-lock headers', + () => { + assert.deepStrictEqual(putBucketApiMethods({ 'x-amz-bucket-object-lock-enabled': 'true' }), [ + 'bucketPut', + 'bucketPutObjectLock', + 'bucketPutVersioning', + ]); + }, + ); - it('should return s3:CreateBucket and s3:PutBucketAcl' + - ' with ACL headers', () => { - assert.deepStrictEqual( - putBucketApiMethods({ 'x-amz-grant-read': 'private' }), - ['bucketPut', 'bucketPutACL'], - ); + it('should return s3:CreateBucket and s3:PutBucketAcl' + ' with ACL headers', () => { + assert.deepStrictEqual(putBucketApiMethods({ 'x-amz-grant-read': 'private' }), [ + 'bucketPut', + 'bucketPutACL', + ]); }); }); }); diff --git a/tests/unit/testConfigs/bucketNotifConfigTest.js b/tests/unit/testConfigs/bucketNotifConfigTest.js index 9feded3dac..38118dee44 100644 --- a/tests/unit/testConfigs/bucketNotifConfigTest.js +++ b/tests/unit/testConfigs/bucketNotifConfigTest.js @@ -3,13 +3,15 @@ const { bucketNotifAssert } = require('../../../lib/Config'); describe('bucketNotifAssert', () => { it('should not throw an error if bucket notification config is valid', () => { - bucketNotifAssert([{ - resource: 'target1', - type: 'kafka', - host: 'localhost', - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 'kafka', + host: 'localhost', + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }); it('should throw an error if bucket notification config is not an array', () => { assert.throws(() => { @@ -20,51 +22,58 @@ describe('bucketNotifAssert', () => { port: 8000, auth: { user: 'user', password: 'password' }, }); - }, - '/bad config: bucket notification configuration must be an array/'); + }, '/bad config: bucket notification configuration must be an array/'); }); it('should throw an error if resource is not a string', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 12345, - type: 'kafka', - host: 'localhost', - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 12345, + type: 'kafka', + host: 'localhost', + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: bucket notification configuration resource must be a string/'); }); it('should throw an error if type is not a string', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 'target1', - type: 12345, - host: 'localhost', - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 12345, + host: 'localhost', + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: bucket notification configuration type must be a string/'); }); it('should throw an error if host is not a string', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 'target1', - type: 'kafka', - host: 127.0, - port: 8000, - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 'kafka', + host: 127.0, + port: 8000, + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: bucket notification configuration type must be a string/'); }); it('should throw an error if port is not an integer', () => { assert.throws(() => { - bucketNotifAssert([{ - resource: 'target1', - type: 'kafka', - host: 'localhost', - port: '8000', - auth: { user: 'user', password: 'password' }, - }]); + bucketNotifAssert([ + { + resource: 'target1', + type: 'kafka', + host: 'localhost', + port: '8000', + auth: { user: 'user', password: 'password' }, + }, + ]); }, '/bad config: port must be a positive integer/'); }); // TODO: currently auth is fluid and once a concrete structure is diff --git a/tests/unit/testConfigs/configTest.js b/tests/unit/testConfigs/configTest.js index 881b5f9e1f..afd73f092e 100644 --- a/tests/unit/testConfigs/configTest.js +++ b/tests/unit/testConfigs/configTest.js @@ -6,144 +6,185 @@ const { config } = require('../../../lib/Config'); const userBucketOwner = 'Bart'; const creationDate = new Date().toJSON(); -const serverSideEncryption = { cryptoScheme: 123, algorithm: 'algo', -masterKeyId: 'masterKeyId', mandatory: false }; -const bucketOne = new BucketInfo('bucketone', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion()); -const bucketTwo = new BucketInfo('buckettwo', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion()); -const bucketOnetWithEncryption = new BucketInfo('bucketone', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion(), undefined, undefined, undefined, - serverSideEncryption); -const bucketTwoWithEncryption = new BucketInfo('buckettwo', - userBucketOwner, userBucketOwner, creationDate, - BucketInfo.currentModelVersion(), undefined, undefined, undefined, - serverSideEncryption); +const serverSideEncryption = { cryptoScheme: 123, algorithm: 'algo', masterKeyId: 'masterKeyId', mandatory: false }; +const bucketOne = new BucketInfo( + 'bucketone', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), +); +const bucketTwo = new BucketInfo( + 'buckettwo', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), +); +const bucketOnetWithEncryption = new BucketInfo( + 'bucketone', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), + undefined, + undefined, + undefined, + serverSideEncryption, +); +const bucketTwoWithEncryption = new BucketInfo( + 'buckettwo', + userBucketOwner, + userBucketOwner, + creationDate, + BucketInfo.currentModelVersion(), + undefined, + undefined, + undefined, + serverSideEncryption, +); const results = [ - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend2', - destLocationConstraintName: 'azurebackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend', - destLocationConstraintName: 'awsbackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend', - destLocationConstraintName: 'awsbackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend2', - destLocationConstraintName: 'awsbackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: true, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'scality-internal-mem', - destLocationConstraintName: 'scality-internal-mem', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'scality-internal-mem', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'scality-internal-mem', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'awsbackend', - destLocationConstraintName: 'scality-internal-mem', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'scality-internal-mem', - destLocationConstraintName: 'awsbackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'awsbackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend2', - sourceBucketMD: bucketOne, - destBucketMD: bucketOne, - boolExpected: false, - description: 'same bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOne, - destBucketMD: bucketTwo, - boolExpected: true, - description: 'different non-encrypted bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOnetWithEncryption, - destBucketMD: bucketOnetWithEncryption, - boolExpected: true, - description: 'same encrypted bucket metadata', - }, - { sourceLocationConstraintName: 'azurebackend', - destLocationConstraintName: 'azurebackend', - sourceBucketMD: bucketOnetWithEncryption, - destBucketMD: bucketTwoWithEncryption, - boolExpected: false, - description: 'different encrypted bucket metadata', - }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend2', + destLocationConstraintName: 'azurebackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend', + destLocationConstraintName: 'awsbackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend', + destLocationConstraintName: 'awsbackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend2', + destLocationConstraintName: 'awsbackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: true, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'scality-internal-mem', + destLocationConstraintName: 'scality-internal-mem', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'scality-internal-mem', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'scality-internal-mem', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'awsbackend', + destLocationConstraintName: 'scality-internal-mem', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'scality-internal-mem', + destLocationConstraintName: 'awsbackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'awsbackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend2', + sourceBucketMD: bucketOne, + destBucketMD: bucketOne, + boolExpected: false, + description: 'same bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOne, + destBucketMD: bucketTwo, + boolExpected: true, + description: 'different non-encrypted bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOnetWithEncryption, + destBucketMD: bucketOnetWithEncryption, + boolExpected: true, + description: 'same encrypted bucket metadata', + }, + { + sourceLocationConstraintName: 'azurebackend', + destLocationConstraintName: 'azurebackend', + sourceBucketMD: bucketOnetWithEncryption, + destBucketMD: bucketTwoWithEncryption, + boolExpected: false, + description: 'different encrypted bucket metadata', + }, ]; describe('Testing Config.js function: ', () => { results.forEach(result => { - it(`should return ${result.boolExpected} if source location ` + - `constraint === ${result.sourceLocationConstraintName} ` + - 'and destination location constraint ===' + - ` ${result.destLocationConstraintName} and ${result.description}`, - done => { - const isCopy = utils.externalBackendCopy(config, - result.sourceLocationConstraintName, - result.destLocationConstraintName, result.sourceBucketMD, - result.destBucketMD); - assert.strictEqual(isCopy, result.boolExpected); - done(); - }); + it( + `should return ${result.boolExpected} if source location ` + + `constraint === ${result.sourceLocationConstraintName} ` + + 'and destination location constraint ===' + + ` ${result.destLocationConstraintName} and ${result.description}`, + done => { + const isCopy = utils.externalBackendCopy( + config, + result.sourceLocationConstraintName, + result.destLocationConstraintName, + result.sourceBucketMD, + result.destBucketMD, + ); + assert.strictEqual(isCopy, result.boolExpected); + done(); + }, + ); }); }); diff --git a/tests/unit/testConfigs/locConstraintAssert.js b/tests/unit/testConfigs/locConstraintAssert.js index f56fbec6e5..e9051d4b31 100644 --- a/tests/unit/testConfigs/locConstraintAssert.js +++ b/tests/unit/testConfigs/locConstraintAssert.js @@ -7,23 +7,30 @@ class LocationConstraint { this.objectId = objectId; this.legacyAwsBehavior = legacyAwsBehavior || false; this.sizeLimitGB = sizeLimit || undefined; - this.details = Object.assign({}, { - awsEndpoint: 's3.amazonaws.com', - bucketName: 'tester', - credentialsProfile: 'default', - region: 'us-west-1', - }, details || {}); + this.details = Object.assign( + {}, + { + awsEndpoint: 's3.amazonaws.com', + bucketName: 'tester', + credentialsProfile: 'default', + region: 'us-west-1', + }, + details || {}, + ); } } function getAzureDetails(replaceParams) { - return Object.assign({ - azureStorageEndpoint: 'https://fakeaccountname.blob.core.fake.net/', - azureStorageAccountName: 'fakeaccountname', - azureStorageAccessKey: 'Fake00Key123', - bucketMatch: false, - azureContainerName: 'test', - }, replaceParams); + return Object.assign( + { + azureStorageEndpoint: 'https://fakeaccountname.blob.core.fake.net/', + azureStorageAccountName: 'fakeaccountname', + azureStorageAccessKey: 'Fake00Key123', + bucketMatch: false, + azureContainerName: 'test', + }, + replaceParams, + ); } // FIXME: most of tests using a line-wrapped regexp are broken, @@ -35,331 +42,301 @@ describe('locationConstraintAssert', () => { it('should throw error if locationConstraints is not an object', () => { assert.throws(() => { locationConstraintAssert(''); - }, - /bad config: locationConstraints must be an object/); + }, /bad config: locationConstraints must be an object/); }); it('should throw error if any location constraint is not an object', () => { - assert.throws(() => { - locationConstraintAssert({ notObject: '' }); - }, - err => { - assert.strictEqual(err.message, 'bad config: ' + - 'locationConstraints[region] must be an object'); - return true; - }); + assert.throws( + () => { + locationConstraintAssert({ notObject: '' }); + }, + err => { + assert.strictEqual(err.message, 'bad config: ' + 'locationConstraints[region] must be an object'); + return true; + }, + ); }); it('should throw error if type is not a string', () => { const locationConstraint = new LocationConstraint(42, 'locId'); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].type is mandatory/ + - /and must be a string/); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].type is mandatory/ + /and must be a string/, + ); }); it('should throw error if type is not mem/file/scality/dmf/crr', () => { - const locationConstraint = new LocationConstraint( - 'notSupportedType', 'locId'); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].type must be/ + - /one of mem,file,scality,tlp,crr/); + const locationConstraint = new LocationConstraint('notSupportedType', 'locId'); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].type must be/ + /one of mem,file,scality,tlp,crr/, + ); }); it('should throw error if legacyAwsBehavior is not a boolean', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', 42); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].legacyAwsBehavior / + - /is mandatory and must be a boolean/); + const locationConstraint = new LocationConstraint('scality', 'locId', 42); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].legacyAwsBehavior / + /is mandatory and must be a boolean/, + ); }); it('should throw error if details is not an object', () => { - const locationConstraint = - new LocationConstraint('scality', 'locId', false, 42); - assert.throws(() => { - locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: locationConstraints[region].details is / + - /mandatory and must be an object/); + const locationConstraint = new LocationConstraint('scality', 'locId', false, 42); + assert.throws( + () => { + locationConstraintAssert({ 'scality-east': locationConstraint }); + }, + /bad config: locationConstraints[region].details is / + /mandatory and must be an object/, + ); }); it('should throw error if awsEndpoint is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: awsEndpoint must be a string/); + }, /bad config: awsEndpoint must be a string/); }); it('should throw error if bucketName is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 's3.amazonaws.com', - bucketName: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 's3.amazonaws.com', + bucketName: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: bucketName must be a string/); + }, /bad config: bucketName must be a string/); }); it('should throw error if credentialsProfile is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 's3.amazonaws.com', - bucketName: 'premadebucket', - credentialsProfile: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 's3.amazonaws.com', + bucketName: 'premadebucket', + credentialsProfile: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: credentialsProfile must be a string/); + }, /bad config: credentialsProfile must be a string/); }); it('should throw error if region is not a string', () => { - const locationConstraint = new LocationConstraint( - 'scality', 'locId', false, - { - awsEndpoint: 's3.amazonaws.com', - bucketName: 'premadebucket', - credentialsProfile: 'zenko', - region: 42, - }); + const locationConstraint = new LocationConstraint('scality', 'locId', false, { + awsEndpoint: 's3.amazonaws.com', + bucketName: 'premadebucket', + credentialsProfile: 'zenko', + region: 42, + }); assert.throws(() => { locationConstraintAssert({ 'scality-east': locationConstraint }); - }, - /bad config: region must be a string/); + }, /bad config: region must be a string/); }); it('should throw error if us-east-1 not specified', () => { const locationConstraint = new LocationConstraint(); assert.throws(() => { locationConstraintAssert({ 'not-us-east-1': locationConstraint }); - }, - '/bad locationConfig: must ' + - 'include us-east-1 as a locationConstraint/'); + }, '/bad locationConfig: must ' + 'include us-east-1 as a locationConstraint/'); }); it('should not throw error for a valid azure location constraint', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails()); + const locationConstraint = new LocationConstraint('azure', 'locId2', true, getAzureDetails()); assert.doesNotThrow(() => { - locationConstraintAssert({ 'azurefaketest': locationConstraint, - 'us-east-1': usEast1 }); - }, - '/should not throw for a valid azure location constraint/'); + locationConstraintAssert({ azurefaketest: locationConstraint, 'us-east-1': usEast1 }); + }, '/should not throw for a valid azure location constraint/'); }); - it('should throw error if type is azure and azureContainerName is ' + - 'not specified', () => { + it('should throw error if type is azure and azureContainerName is ' + 'not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureContainerName: undefined })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureContainerName: undefined }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: ' + - '"azurefaketest" azureContainerName must be defined/'); + }, '/bad location constraint: ' + '"azurefaketest" azureContainerName must be defined/'); }); - it('should throw error if type is azure and azureContainerName is ' + - 'invalid value', () => { + it('should throw error if type is azure and azureContainerName is ' + 'invalid value', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureContainerName: '.' })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureContainerName: '.' }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureContainerName is an invalid container name/'); + }, '/bad location constraint: "azurefaketest" ' + 'azureContainerName is an invalid container name/'); }); - it('should throw error if type is azure and azureStorageEndpoint ' + - 'is not specified', () => { + it('should throw error if type is azure and azureStorageEndpoint ' + 'is not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageEndpoint: undefined })); - assert.throws(() => { - locationConstraintAssert({ - 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, - }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageEndpoint must be set in locationConfig ' + - 'or environment variable/'); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageEndpoint: undefined }), + ); + assert.throws( + () => { + locationConstraintAssert({ + 'us-east-1': usEast1, + azurefaketest: locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageEndpoint must be set in locationConfig ' + + 'or environment variable/', + ); }); - it('should throw error if type is azure and azureStorageAccountName ' + - 'is not specified', () => { + it('should throw error if type is azure and azureStorageAccountName ' + 'is not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccountName: undefined })); - assert.throws(() => { - locationConstraintAssert({ - 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, - }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccountName must be set in locationConfig ' + - 'or environment variable/'); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccountName: undefined }), + ); + assert.throws( + () => { + locationConstraintAssert({ + 'us-east-1': usEast1, + azurefaketest: locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccountName must be set in locationConfig ' + + 'or environment variable/', + ); }); - it('should throw error if type is azure and azureStorageAccountName ' + - 'is invalid value', () => { + it('should throw error if type is azure and azureStorageAccountName ' + 'is invalid value', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccountName: 'invalid!!!' })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccountName: 'invalid!!!' }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccountName "invalid!!!" is an invalid value/'); + }, '/bad location constraint: "azurefaketest" ' + 'azureStorageAccountName "invalid!!!" is an invalid value/'); }); - it('should throw error if type is azure and azureStorageAccessKey ' + - 'is not specified', () => { + it('should throw error if type is azure and azureStorageAccessKey ' + 'is not specified', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccessKey: undefined })); - assert.throws(() => { - locationConstraintAssert({ - 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, - }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccessKey must be set in locationConfig ' + - 'or environment variable/'); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccessKey: undefined }), + ); + assert.throws( + () => { + locationConstraintAssert({ + 'us-east-1': usEast1, + azurefaketest: locationConstraint, + }); + }, + '/bad location constraint: "azurefaketest" ' + + 'azureStorageAccessKey must be set in locationConfig ' + + 'or environment variable/', + ); }); - it('should throw error if type is azure and azureStorageAccessKey ' + - 'is not a valid base64 string', () => { + it('should throw error if type is azure and azureStorageAccessKey ' + 'is not a valid base64 string', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); const locationConstraint = new LocationConstraint( - 'azure', 'locId2', true, - getAzureDetails({ azureStorageAccessKey: 'invalid!!!' })); + 'azure', + 'locId2', + true, + getAzureDetails({ azureStorageAccessKey: 'invalid!!!' }), + ); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'azurefaketest': locationConstraint, + azurefaketest: locationConstraint, }); - }, - '/bad location constraint: "azurefaketest" ' + - 'azureStorageAccessKey is not a valid base64 string/'); + }, '/bad location constraint: "azurefaketest" ' + 'azureStorageAccessKey is not a valid base64 string/'); }); it('should set https to true by default', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awshttpsDefault': locationConstraint, + awshttpsDefault: locationConstraint, }); - }, '/bad location constraint awshttpsDefault,' + - 'incorrect default config for https'); - assert.strictEqual(locationConstraint.details.https, true, - 'https config should be true'); + }, '/bad location constraint awshttpsDefault,' + 'incorrect default config for https'); + assert.strictEqual(locationConstraint.details.https, true, 'https config should be true'); }); it('should override default if https is set to false', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true, { - https: false, - }); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true, { + https: false, + }); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awshttpsFalse': locationConstraint, + awshttpsFalse: locationConstraint, }); - }, '/bad location constraint awshttpsFalse,' + - 'incorrect config for https'); - assert.strictEqual(locationConstraint.details.https, false, - 'https config should be false'); + }, '/bad location constraint awshttpsFalse,' + 'incorrect config for https'); + assert.strictEqual(locationConstraint.details.https, false, 'https config should be false'); }); it('should set pathStyle config option to false by default', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awsdefaultstyle': locationConstraint, + awsdefaultstyle: locationConstraint, }); }, '/bad location constraint, unable to set default config'); - assert.strictEqual(locationConstraint.details.pathStyle, false, - 'pathstyle config should be false'); + assert.strictEqual(locationConstraint.details.pathStyle, false, 'pathstyle config should be false'); }); it('should override default if pathStyle is set to true', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true, - { pathStyle: true }); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true, { pathStyle: true }); assert.doesNotThrow(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awspathstyle': locationConstraint, + awspathstyle: locationConstraint, }); }, '/bad location constraint, unable to set pathSytle config'); - assert.strictEqual(locationConstraint.details.pathStyle, true, - 'pathstyle config should be true'); + assert.strictEqual(locationConstraint.details.pathStyle, true, 'pathstyle config should be true'); }); it('should throw error if sizeLimitGB is not a number', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'aws_s3', 'locId2', true, - null, true); + const locationConstraint = new LocationConstraint('aws_s3', 'locId2', true, null, true); assert.throws(() => { locationConstraintAssert({ 'us-east-1': usEast1, - 'awsstoragesizelimit': locationConstraint, + awsstoragesizelimit: locationConstraint, }); - }, - '/bad config: locationConstraints[region].sizeLimitGB ' + - 'must be a number (in gigabytes)'); + }, '/bad config: locationConstraints[region].sizeLimitGB ' + 'must be a number (in gigabytes)'); }); it('should throw error if objectId is not set', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'azure', undefined, true, - getAzureDetails()); + const locationConstraint = new LocationConstraint('azure', undefined, true, getAzureDetails()); assert.throws(() => { - locationConstraintAssert({ 'azurefaketest': locationConstraint, - 'us-east-1': usEast1 }); - }, - '/bad config: locationConstraints[region].objectId is mandatory ' + - 'and must be a unique string across locations'); + locationConstraintAssert({ azurefaketest: locationConstraint, 'us-east-1': usEast1 }); + }, '/bad config: locationConstraints[region].objectId is mandatory ' + 'and must be a unique string across locations'); }); it('should throw error if objectId is duplicated', () => { const usEast1 = new LocationConstraint(undefined, 'locId1'); - const locationConstraint = new LocationConstraint( - 'azure', 'locId1', true, - getAzureDetails()); + const locationConstraint = new LocationConstraint('azure', 'locId1', true, getAzureDetails()); assert.throws(() => { - locationConstraintAssert({ 'azurefaketest': locationConstraint, - 'us-east-1': usEast1 }); - }, - '/bad config: location constraint objectId "locId1" is not unique ' + - 'across configured locations'); + locationConstraintAssert({ azurefaketest: locationConstraint, 'us-east-1': usEast1 }); + }, '/bad config: location constraint objectId "locId1" is not unique ' + 'across configured locations'); }); }); diff --git a/tests/unit/testConfigs/parseKmsAWS.js b/tests/unit/testConfigs/parseKmsAWS.js index c075375192..c466bbee77 100644 --- a/tests/unit/testConfigs/parseKmsAWS.js +++ b/tests/unit/testConfigs/parseKmsAWS.js @@ -338,6 +338,6 @@ describe('parseKmsAWS TLS section', () => { assert(readFileSyncStub.calledWith(path.join(basePath, caPath))); }); - assert(readFileSyncStub.callCount === (keyPaths.length + certPaths.length + caPaths.length)); + assert(readFileSyncStub.callCount === keyPaths.length + certPaths.length + caPaths.length); }); }); diff --git a/tests/unit/testConfigs/parseRedisConfig.spec.js b/tests/unit/testConfigs/parseRedisConfig.spec.js index f585bcec7a..2583cc266c 100644 --- a/tests/unit/testConfigs/parseRedisConfig.spec.js +++ b/tests/unit/testConfigs/parseRedisConfig.spec.js @@ -31,8 +31,7 @@ describe('parseRedisConfig', () => { input: { host: 'localhost', port: 6479, - retry: { - }, + retry: {}, }, }, { @@ -195,8 +194,7 @@ describe('parseRedisConfig', () => { host: 'localhost', port: 6479, retry: { - connectBackoff: { - }, + connectBackoff: {}, }, }, }, diff --git a/tests/unit/testConfigs/parseSproxydConfig.js b/tests/unit/testConfigs/parseSproxydConfig.js index ae22ba94c6..4f057a7771 100644 --- a/tests/unit/testConfigs/parseSproxydConfig.js +++ b/tests/unit/testConfigs/parseSproxydConfig.js @@ -19,16 +19,14 @@ function makeSproxydConf(bootstrap, chordCos, sproxydPath) { describe('parseSproxydConfig', () => { it('should return a parsed config if valid', () => { - const sproxydConf = parseSproxydConfig(makeSproxydConf( - ['localhost:8181'], null, '/arc')); + const sproxydConf = parseSproxydConfig(makeSproxydConf(['localhost:8181'], null, '/arc')); assert.deepStrictEqual(sproxydConf, { bootstrap: ['localhost:8181'], path: '/arc', }); }); it('should return a parsed config with chordCos if valid', () => { - const sproxydConf = parseSproxydConfig(makeSproxydConf( - ['localhost:8181'], '3', '/arc')); + const sproxydConf = parseSproxydConfig(makeSproxydConf(['localhost:8181'], '3', '/arc')); assert.deepStrictEqual(sproxydConf, { bootstrap: ['localhost:8181'], path: '/arc', @@ -40,8 +38,7 @@ describe('parseSproxydConfig', () => { parseSproxydConfig(makeSproxydConf('localhost:8181')); }); }); - it('should throw an error if bootstrap array does not contain strings', - () => { + it('should throw an error if bootstrap array does not contain strings', () => { assert.throws(() => { parseSproxydConfig(makeSproxydConf([8181])); }); diff --git a/tests/unit/testConfigs/requestsConfigTest.js b/tests/unit/testConfigs/requestsConfigTest.js index 41dbdee516..9569534a33 100644 --- a/tests/unit/testConfigs/requestsConfigTest.js +++ b/tests/unit/testConfigs/requestsConfigTest.js @@ -5,11 +5,9 @@ describe('requestsConfigAssert', () => { it('should not throw an error if there is no requests config', () => { assert.doesNotThrow(() => { requestsConfigAssert({}); - }, - 'should not throw an error if there is no requests config'); + }, 'should not throw an error if there is no requests config'); }); - it('should not throw an error if requests config via proxy is set to false', - () => { + it('should not throw an error if requests config via proxy is set to false', () => { assert.doesNotThrow(() => { requestsConfigAssert({ viaProxy: false, @@ -17,25 +15,24 @@ describe('requestsConfigAssert', () => { extractClientIPFromHeader: '', extractProtocolFromHeader: '', }); - }, - 'shouldnt throw an error if requests config via proxy is set to false'); + }, 'shouldnt throw an error if requests config via proxy is set to false'); }); - it('should not throw an error if requests config via proxy is true, ' + - 'trustedProxyCIDRs & extractClientIPFromHeader & ' + - 'extractProtocolFromHeader are set', () => { - assert.doesNotThrow(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: ['123.123.123.123'], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 'x-forwarded-proto', - }); + it( + 'should not throw an error if requests config via proxy is true, ' + + 'trustedProxyCIDRs & extractClientIPFromHeader & ' + + 'extractProtocolFromHeader are set', + () => { + assert.doesNotThrow(() => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: ['123.123.123.123'], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, 'should not throw an error if requests config ' + 'via proxy is set correctly'); }, - 'should not throw an error if requests config ' + - 'via proxy is set correctly'); - }); - it('should throw an error if requests.viaProxy is not a boolean', - () => { + ); + it('should throw an error if requests.viaProxy is not a boolean', () => { assert.throws(() => { requestsConfigAssert({ viaProxy: 1, @@ -43,93 +40,97 @@ describe('requestsConfigAssert', () => { extractClientIPFromHeader: 'x-forwarded-for', extractProtocolFromHeader: 'x-forwarded-proto', }); - }, - '/config: invalid requests configuration. viaProxy must be a ' + - 'boolean/'); + }, '/config: invalid requests configuration. viaProxy must be a ' + 'boolean/'); }); - it('should throw an error if requests.trustedProxyCIDRs is not an array', - () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: 1, - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'trustedProxyCIDRs must be set if viaProxy is set to true ' + - 'and must be an array/'); + it('should throw an error if requests.trustedProxyCIDRs is not an array', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: 1, + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'trustedProxyCIDRs must be set if viaProxy is set to true ' + + 'and must be an array/', + ); }); - it('should throw an error if requests.trustedProxyCIDRs array is empty', - () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'trustedProxyCIDRs must be set if viaProxy is set to true ' + - 'and must be an array/'); + it('should throw an error if requests.trustedProxyCIDRs array is empty', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'trustedProxyCIDRs must be set if viaProxy is set to true ' + + 'and must be an array/', + ); }); - it('should throw an error if requests.extractClientIPFromHeader ' + - 'is not a string', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 1, - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'extractClientIPFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractClientIPFromHeader ' + 'is not a string', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 1, + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'extractClientIPFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); - it('should throw an error if requests.extractProtocolFromHeader ' + - 'is not a string', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: 1, - }); - }, - '/config: invalid requests configuration. ' + - 'extractProtocolFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractProtocolFromHeader ' + 'is not a string', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: 1, + }); + }, + '/config: invalid requests configuration. ' + + 'extractProtocolFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); - it('should throw an error if requests.extractClientIPFromHeader ' + - 'is empty', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: '', - extractProtocolFromHeader: 'x-forwarded-proto', - }); - }, - '/config: invalid requests configuration. ' + - 'extractClientIPFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractClientIPFromHeader ' + 'is empty', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: '', + extractProtocolFromHeader: 'x-forwarded-proto', + }); + }, + '/config: invalid requests configuration. ' + + 'extractClientIPFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); - it('should throw an error if requests.extractProtocolFromHeader ' + - 'is empty', () => { - assert.throws(() => { - requestsConfigAssert({ - viaProxy: true, - trustedProxyCIDRs: [], - extractClientIPFromHeader: 'x-forwarded-for', - extractProtocolFromHeader: '', - }); - }, - '/config: invalid requests configuration. ' + - 'extractProtocolFromHeader must be set if viaProxy is ' + - 'set to true and must be a string/'); + it('should throw an error if requests.extractProtocolFromHeader ' + 'is empty', () => { + assert.throws( + () => { + requestsConfigAssert({ + viaProxy: true, + trustedProxyCIDRs: [], + extractClientIPFromHeader: 'x-forwarded-for', + extractProtocolFromHeader: '', + }); + }, + '/config: invalid requests configuration. ' + + 'extractProtocolFromHeader must be set if viaProxy is ' + + 'set to true and must be a string/', + ); }); it('should lowercase the extractClientIPFromHeader and extractProtocolFromHeader values', () => { const config = { diff --git a/tests/unit/utils/aclUtils.js b/tests/unit/utils/aclUtils.js index 9985ca8af1..c0b397b2a4 100644 --- a/tests/unit/utils/aclUtils.js +++ b/tests/unit/utils/aclUtils.js @@ -1,39 +1,33 @@ const assert = require('assert'); const aclUtils = require('../../../lib/utilities/aclUtils'); - describe('checkGrantHeaderValidity for acls', () => { const tests = [ { it: 'should allow valid x-amz-grant-read grant', headers: { - 'x-amz-grant-read': - 'uri=http://acs.amazonaws.com/groups/global/AllUsers', + 'x-amz-grant-read': 'uri=http://acs.amazonaws.com/groups/global/AllUsers', }, result: true, }, { it: 'should allow valid x-amz-grant-write grant', headers: { - 'x-amz-grant-write': - 'emailaddress=user2@example.com', + 'x-amz-grant-write': 'emailaddress=user2@example.com', }, result: true, }, { it: 'should allow valid x-amz-grant-read-acp grant', headers: { - 'x-amz-grant-read-acp': - 'emailaddress=superuser@example.com', + 'x-amz-grant-read-acp': 'emailaddress=superuser@example.com', }, result: true, }, { it: 'should allow valid x-amz-grant-write-acp grant', headers: { - 'x-amz-grant-write-acp': - 'id=79a59df900b949e55d96a1e6' + - '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + 'x-amz-grant-write-acp': 'id=79a59df900b949e55d96a1e6' + '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, result: true, }, @@ -41,50 +35,44 @@ describe('checkGrantHeaderValidity for acls', () => { it: 'should allow valid x-amz-grant-full-control grant', headers: { 'x-amz-grant-full-control': - 'id=79a59df900b949e55d96a1e6' + - '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be,' + - 'emailaddress=foo@bar.com', + 'id=79a59df900b949e55d96a1e6' + + '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be,' + + 'emailaddress=foo@bar.com', }, result: true, }, { it: 'should deny grant without equal sign', headers: { - 'x-amz-grant-full-control': - 'id79a59df900b949e55d96a1e6' + - '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', + 'x-amz-grant-full-control': 'id79a59df900b949e55d96a1e6' + '98fbacedfd6e09d98eacf8f8d5218e7cd47ef2be', }, result: false, }, { it: 'should deny grant with bad uri', headers: { - 'x-amz-grant-full-control': - 'uri=http://totallymadeup', + 'x-amz-grant-full-control': 'uri=http://totallymadeup', }, result: false, }, { it: 'should deny grant with bad emailaddress', headers: { - 'x-amz-grant-read': - 'emailaddress=invalidemail.com', + 'x-amz-grant-read': 'emailaddress=invalidemail.com', }, result: false, }, { it: 'should deny grant with bad canonicalID', headers: { - 'x-amz-grant-write': - 'id=123', + 'x-amz-grant-write': 'id=123', }, result: false, }, { it: 'should deny grant with bad type of identifier', headers: { - 'x-amz-grant-write': - 'madeupidentifier=123', + 'x-amz-grant-write': 'madeupidentifier=123', }, result: false, }, @@ -92,8 +80,7 @@ describe('checkGrantHeaderValidity for acls', () => { tests.forEach(test => { it(test.it, () => { - const actualResult = - aclUtils.checkGrantHeaderValidity(test.headers); + const actualResult = aclUtils.checkGrantHeaderValidity(test.headers); assert.strictEqual(actualResult, test.result); }); }); diff --git a/tests/unit/utils/bucketEncryption.js b/tests/unit/utils/bucketEncryption.js index 63cb33df9a..230f18f891 100644 --- a/tests/unit/utils/bucketEncryption.js +++ b/tests/unit/utils/bucketEncryption.js @@ -1,14 +1,12 @@ const metadata = require('../../../lib/metadata/wrapper'); - function templateSSEConfig({ algorithm, keyId }) { const xml = []; xml.push(` - ` - ); + `); if (algorithm) { xml.push(`${algorithm}`); diff --git a/tests/unit/utils/checkReadLocation.js b/tests/unit/utils/checkReadLocation.js index 6756f9bff4..34ce3fd712 100644 --- a/tests/unit/utils/checkReadLocation.js +++ b/tests/unit/utils/checkReadLocation.js @@ -1,8 +1,7 @@ const assert = require('assert'); const { ConfigObject } = require('../../../lib/Config'); -const checkReadLocation = - require('../../../lib/api/apiUtils/object/checkReadLocation'); +const checkReadLocation = require('../../../lib/api/apiUtils/object/checkReadLocation'); const locationConstraints = { bucketmatch: { @@ -38,14 +37,12 @@ describe('Testing checkReadLocation', () => { }); it('should return null if location does not exist', () => { - const testResult = checkReadLocation( - config, 'nonexistloc', key, bucket); + const testResult = checkReadLocation(config, 'nonexistloc', key, bucket); assert.deepStrictEqual(testResult, null); }); it('should return correct results for bucketMatch true location', () => { - const testResult = checkReadLocation( - config, 'bucketmatch', key, bucket); + const testResult = checkReadLocation(config, 'bucketmatch', key, bucket); const expectedResult = { location: 'bucketmatch', key, @@ -55,8 +52,7 @@ describe('Testing checkReadLocation', () => { }); it('should return correct results for bucketMatch false location', () => { - const testResult = checkReadLocation( - config, 'nobucketmatch', key, bucket); + const testResult = checkReadLocation(config, 'nobucketmatch', key, bucket); const expectedResult = { location: 'nobucketmatch', key: `${bucket}/${key}`, diff --git a/tests/unit/utils/gcpMpuHelpers.js b/tests/unit/utils/gcpMpuHelpers.js index 99496a6a3d..6820dda281 100644 --- a/tests/unit/utils/gcpMpuHelpers.js +++ b/tests/unit/utils/gcpMpuHelpers.js @@ -39,8 +39,7 @@ describe('GcpUtils MPU Helper Functions:', () => { tests.forEach(test => { it(test.it, () => { const { partNumber, phase } = test.input; - assert.strictEqual(createMpuKey( - key, uploadId, partNumber, phase), test.output); + assert.strictEqual(createMpuKey(key, uploadId, partNumber, phase), test.output); }); }); }); @@ -56,9 +55,7 @@ describe('GcpUtils MPU Helper Functions:', () => { tests.forEach(test => { it(test.it, () => { const { phase, size } = test.input; - assert.deepStrictEqual(createMpuList( - { Key: key, UploadId: uploadId }, phase, size), - test.output); + assert.deepStrictEqual(createMpuList({ Key: key, UploadId: uploadId }, phase, size), test.output); }); }); }); diff --git a/tests/unit/utils/gcpTaggingHelpers.js b/tests/unit/utils/gcpTaggingHelpers.js index d2169156d0..b7018bba42 100644 --- a/tests/unit/utils/gcpTaggingHelpers.js +++ b/tests/unit/utils/gcpTaggingHelpers.js @@ -1,10 +1,8 @@ const assert = require('assert'); const { errorInstances, storage } = require('arsenal'); const { gcpTaggingPrefix } = require('../../../constants'); -const { genPutTagObj } = - require('../../../tests/functional/raw-node/utils/gcpUtils'); -const { processTagSet, stripTags, retrieveTags, getPutTagsMetadata } = - storage.data.external.GcpUtils; +const { genPutTagObj } = require('../../../tests/functional/raw-node/utils/gcpUtils'); +const { processTagSet, stripTags, retrieveTags, getPutTagsMetadata } = storage.data.external.GcpUtils; const maxTagSize = 10; const validTagSet = genPutTagObj(2); @@ -36,26 +34,24 @@ describe('GcpUtils Tagging Helper Functions:', () => { { it: 'should return error for invalid tag set size', input: invalidSizeTagSet, - output: errorInstances.BadRequest.customizeDescription( - 'Object tags cannot be greater than 10'), + output: errorInstances.BadRequest.customizeDescription('Object tags cannot be greater than 10'), }, { it: 'should return error for duplicate tag keys', input: invalidDuplicateTagSet, output: errorInstances.InvalidTag.customizeDescription( - 'Cannot provide multiple Tags with the same key'), + 'Cannot provide multiple Tags with the same key', + ), }, { it: 'should return error for invalid "key" value', input: invalidKeyTagSet, - output: errorInstances.InvalidTag.customizeDescription( - 'The TagKey provided is too long, 129'), + output: errorInstances.InvalidTag.customizeDescription('The TagKey provided is too long, 129'), }, { it: 'should return error for invalid "value" value', input: invalidValueTagSet, - output: errorInstances.InvalidTag.customizeDescription( - 'The TagValue provided is too long, 257'), + output: errorInstances.InvalidTag.customizeDescription('The TagValue provided is too long, 257'), }, { it: 'should return empty tag object when input is undefined', @@ -123,8 +119,7 @@ describe('GcpUtils Tagging Helper Functions:', () => { describe('getPutTagsMetadata', () => { const tests = [ { - it: 'should return correct object when' + - ' given a tag query string and a metadata obj', + it: 'should return correct object when' + ' given a tag query string and a metadata obj', input: { metadata: Object.assign({}, onlyMetadata), tagQuery }, output: tagMetadata, }, @@ -139,7 +134,8 @@ describe('GcpUtils Tagging Helper Functions:', () => { output: onlyMetadata, }, { - it: 'should return metadata with correct tag properties ' + + it: + 'should return metadata with correct tag properties ' + 'if given a metdata with prior tags and query string', input: { metadata: Object.assign({}, withPriorTags), tagQuery }, output: tagMetadata, @@ -148,8 +144,7 @@ describe('GcpUtils Tagging Helper Functions:', () => { tests.forEach(test => { it(test.it, () => { const { metadata, tagQuery } = test.input; - assert.deepStrictEqual( - getPutTagsMetadata(metadata, tagQuery), test.output); + assert.deepStrictEqual(getPutTagsMetadata(metadata, tagQuery), test.output); }); }); }); diff --git a/tests/unit/utils/lifecycleHelpers.js b/tests/unit/utils/lifecycleHelpers.js index b9399b00ac..a35cff8e4d 100644 --- a/tests/unit/utils/lifecycleHelpers.js +++ b/tests/unit/utils/lifecycleHelpers.js @@ -33,7 +33,8 @@ function getLifecycleXml() { const days2 = 1; const action3 = 'AbortIncompleteMultipartUpload'; const days3 = 30; - return '' + '' + `${id1}` + @@ -61,7 +62,8 @@ function getLifecycleXml() { `${tags[0].value}` + `<${action1}>${days1}` + '' + - ''; + '' + ); } module.exports = { diff --git a/tests/unit/utils/multipleBackendGateway.js b/tests/unit/utils/multipleBackendGateway.js index c62b89bff6..868af60ad5 100644 --- a/tests/unit/utils/multipleBackendGateway.js +++ b/tests/unit/utils/multipleBackendGateway.js @@ -1,9 +1,7 @@ const assert = require('assert'); const { checkExternalBackend } = require('arsenal').storage.data.external.backendUtils; const sinon = require('sinon'); -const awsLocations = [ - 'awsbackend', -]; +const awsLocations = ['awsbackend']; const statusSuccess = { versioningStatus: 'Enabled', @@ -32,37 +30,46 @@ describe('Testing _checkExternalBackend', function describeF() { beforeEach(done => { this.clock = sinon.useFakeTimers({ shouldAdvanceTime: true }); const clients = getClients(true); - return checkExternalBackend(clients, awsLocations, 'aws_s3', false, - externalBackendHealthCheckInterval, done); + return checkExternalBackend(clients, awsLocations, 'aws_s3', false, externalBackendHealthCheckInterval, done); }); afterEach(() => { this.clock.restore(); }); - it('should not refresh response before externalBackendHealthCheckInterval', - done => { + it('should not refresh response before externalBackendHealthCheckInterval', done => { const clients = getClients(false); - return checkExternalBackend(clients, awsLocations, 'aws_s3', - false, externalBackendHealthCheckInterval, (err, res) => { - if (err) { - return done(err); - } - assert.strictEqual(res[0].awsbackend, statusSuccess); - return done(); - }); - }); - - it('should refresh response after externalBackendHealthCheckInterval', - done => { - const clients = getClients(false); - setTimeout(() => { - checkExternalBackend(clients, awsLocations, 'aws_s3', - false, externalBackendHealthCheckInterval, (err, res) => { + return checkExternalBackend( + clients, + awsLocations, + 'aws_s3', + false, + externalBackendHealthCheckInterval, + (err, res) => { if (err) { return done(err); } - assert.strictEqual(res[0].awsbackend, statusFailure); + assert.strictEqual(res[0].awsbackend, statusSuccess); return done(); - }); + }, + ); + }); + + it('should refresh response after externalBackendHealthCheckInterval', done => { + const clients = getClients(false); + setTimeout(() => { + checkExternalBackend( + clients, + awsLocations, + 'aws_s3', + false, + externalBackendHealthCheckInterval, + (err, res) => { + if (err) { + return done(err); + } + assert.strictEqual(res[0].awsbackend, statusFailure); + return done(); + }, + ); }, externalBackendHealthCheckInterval + 1); this.clock.next(); // test faster }); diff --git a/tests/unit/utils/request.js b/tests/unit/utils/request.js index d403660e27..0ddcc4dfe9 100644 --- a/tests/unit/utils/request.js +++ b/tests/unit/utils/request.js @@ -102,10 +102,7 @@ function testHandler(req, res) { case '/raw': return respondWithValue(req, res, ['bitsandbytes']); case '/json': - return respondWithValue(req, res, [ - postJsonStringified.slice(0, 3), - postJsonStringified.slice(3) - ]); + return respondWithValue(req, res, [postJsonStringified.slice(0, 3), postJsonStringified.slice(3)]); case '/post': if (req.method !== 'POST') { return respondWithError(req, res, 405); @@ -124,7 +121,7 @@ function testHandler(req, res) { } function createProxyServer(proto, targetHost, hostname, port, callback) { - const target = new URL(targetHost); + const target = new URL(targetHost); let options = {}; let serverType = http; if (proto === 'https') { @@ -138,10 +135,7 @@ function createProxyServer(proto, targetHost, hostname, port, callback) { proxy.on('connect', (req, clnt) => { const svr = net.connect(target.port, target.hostname, () => { // handle http -> https - clnt.write( - `HTTP/${req.httpVersion} 200 Connection Established\r\n` + - '\r\n' - ); + clnt.write(`HTTP/${req.httpVersion} 200 Connection Established\r\n` + '\r\n'); svr.pipe(clnt); clnt.pipe(svr); }); @@ -157,8 +151,7 @@ function createTestServer(proto, hostname, port, handler, callback) { options = { key: testKey, cert: testCert }; serverType = https; } - const server = serverType.createServer(options, - handler); + const server = serverType.createServer(options, handler); server.on('error', err => { process.stdout.write(`https server: ${err.stack}\n`); process.exit(1); @@ -167,10 +160,7 @@ function createTestServer(proto, hostname, port, handler, callback) { return server; } -[ - 'http', - 'https', -].forEach(protocol => { +['http', 'https'].forEach(protocol => { describe(`test against ${protocol} server`, () => { const hostname = 'localhost'; const testPort = 4242; @@ -185,37 +175,45 @@ function createTestServer(proto, hostname, port, handler, callback) { before(done => { process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0; - async.series([ - next => { - server = createTestServer( - protocol, hostname, testPort, testHandler, next); - }, - next => { - proxyTarget = createTestServer(protocol, hostname, 8081, - (req, res) => res.end('proxyTarget'), next); - }, - next => { - proxyServer = createProxyServer('http', - targetHost, hostname, proxyPort, next); - }, - next => { - sproxyServer = createProxyServer('https', - targetHost, hostname, sproxyPort, next); - }, - ], done); + async.series( + [ + next => { + server = createTestServer(protocol, hostname, testPort, testHandler, next); + }, + next => { + proxyTarget = createTestServer( + protocol, + hostname, + 8081, + (req, res) => res.end('proxyTarget'), + next, + ); + }, + next => { + proxyServer = createProxyServer('http', targetHost, hostname, proxyPort, next); + }, + next => { + sproxyServer = createProxyServer('https', targetHost, hostname, sproxyPort, next); + }, + ], + done, + ); }); after(done => { process.env.NODE_TLS_REJECT_UNAUTHORIZED = 1; - async.series([ - next => server.close(next), - next => proxyTarget.close(next), - next => { - proxyServer.close(); - sproxyServer.close(); - next(); - }, - ], done); + async.series( + [ + next => server.close(next), + next => proxyTarget.close(next), + next => { + proxyServer.close(); + sproxyServer.close(); + next(); + }, + ], + done, + ); }); afterEach(() => { @@ -251,35 +249,34 @@ function createTestServer(proto, hostname, port, handler, callback) { }); it('should return data', done => { - request.request(`${host}/raw`, { json: false }, - (err, res, body) => { - assert.ifError(err); - assert.equal(body, 'bitsandbytes'); - done(); - }); + request.request(`${host}/raw`, { json: false }, (err, res, body) => { + assert.ifError(err); + assert.equal(body, 'bitsandbytes'); + done(); + }); }); it('should convert output to json if "json" flag is set', done => { - request.request(`${host}/json`, { json: true }, - (err, res, body) => { - assert.ifError(err); - assert.deepStrictEqual(body, postJson); - done(); - }); + request.request(`${host}/json`, { json: true }, (err, res, body) => { + assert.ifError(err); + assert.deepStrictEqual(body, postJson); + done(); + }); }); it('should set method to GET if it is missing', done => { - const req = request.request(`${host}`, - (err, res) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(req.method, 'GET'); - done(); - }); + const req = request.request(`${host}`, (err, res) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(req.method, 'GET'); + done(); + }); }); it('should set headers', done => { - const req = request.request(`${host}`, { + const req = request.request( + `${host}`, + { headers: { 'TEST-HEADERS-ONE': 'test-value-one', 'TEST-HEADERS-TWO': 'test-value-two', @@ -293,75 +290,74 @@ function createTestServer(proto, hostname, port, handler, callback) { 'test-headers-two': 'test-value-two', }); done(); - }); + }, + ); }); }); describe('post', () => { it('should post data', done => { - request.post(`${host}/post`, { body: postData }, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(body, '{"body": "post completed"}'); - done(); - }); + request.post(`${host}/post`, { body: postData }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(body, '{"body": "post completed"}'); + done(); + }); }); it('should post with json data', done => { - request.post(`${host}/postjson`, { body: { key: 'value' } }, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(body, '{"body": "post completed"}'); - done(); - }); + request.post(`${host}/postjson`, { body: { key: 'value' } }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(body, '{"body": "post completed"}'); + done(); + }); }); it('should post with empty body', done => { - request.post(`${host}/postempty`, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.equal(body, '{"body": "post completed"}'); - done(); - }); + request.post(`${host}/postempty`, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.equal(body, '{"body": "post completed"}'); + done(); + }); }); it('should post with json data (json response)', done => { - request.post(`${host}/postjson`, - { body: { key: 'value' }, json: true }, - (err, res, body) => { - assert.ifError(err); - assert.equal(res.statusCode, 200); - assert.deepStrictEqual(body, { - body: 'post completed', - }); - done(); + request.post(`${host}/postjson`, { body: { key: 'value' }, json: true }, (err, res, body) => { + assert.ifError(err); + assert.equal(res.statusCode, 200); + assert.deepStrictEqual(body, { + body: 'post completed', }); + done(); + }); }); it('should set content-type JSON if missing', done => { - const req = request.post(`${host}`, { + const req = request.post( + `${host}`, + { body: postJson, - headers: { 'EXTRA': 'header' }, + headers: { EXTRA: 'header' }, }, (err, res) => { assert.ifError(err); assert.equal(res.statusCode, 200); checkForHeaders(req.getHeaders(), { 'content-type': 'application/json', - 'content-length': - Buffer.byteLength(postJsonStringified), - 'extra': 'header', + 'content-length': Buffer.byteLength(postJsonStringified), + extra: 'header', }); done(); - }); + }, + ); }); - it('should not overwrite existing content-type header value', - done => { - const req = request.post(`${host}`, { + it('should not overwrite existing content-type header value', done => { + const req = request.post( + `${host}`, + { body: postJson, headers: { 'Content-Type': 'text/plain' }, }, @@ -370,22 +366,22 @@ function createTestServer(proto, hostname, port, handler, callback) { assert.equal(res.statusCode, 200); checkForHeaders(req.getHeaders(), { 'content-type': 'text/plain', - 'content-length': - Buffer.byteLength(postJsonStringified), + 'content-length': Buffer.byteLength(postJsonStringified), }); done(); - }); - }); + }, + ); + }); }); }); }); describe('utilities::request error handling', () => { - it('should throw an error if arguments are missing', () => { + it('should throw an error if arguments are missing', () => { assert.throws(request.request); }); - it('should throw an error if callback argument is missing', () => { + it('should throw an error if callback argument is missing', () => { assert.throws(() => request.request('http://test')); }); @@ -427,7 +423,7 @@ describe('utilities::createHeaders', () => { { 'content-type': 'test/one', 'content-length': 1, - } + }, ); }); }); diff --git a/tests/unit/utils/responseStreamData.js b/tests/unit/utils/responseStreamData.js index 10066bfc42..f21861955d 100644 --- a/tests/unit/utils/responseStreamData.js +++ b/tests/unit/utils/responseStreamData.js @@ -8,8 +8,7 @@ const { config } = require('../../../lib/Config'); const { client, implName, data } = require('../../../lib/data/wrapper'); const kms = require('../../../lib/kms/wrapper'); const vault = require('../../../lib/auth/vault'); -const locationStorageCheck = - require('../../../lib/api/apiUtils/object/locationStorageCheck'); +const locationStorageCheck = require('../../../lib/api/apiUtils/object/locationStorageCheck'); const metadata = require('../../../lib/metadata/wrapper'); const routesUtils = s3routes.routesUtils; @@ -51,10 +50,12 @@ describe.skip('responseStreamData:', () => { it('should stream full requested object data for one part object', done => { ds.push(null, dataStoreEntry); - const dataLocations = [{ - key: 1, - dataStore: 'mem', - }]; + const dataLocations = [ + { + key: 1, + dataStore: 'mem', + }, + ]; const response = httpMocks.createResponse({ eventEmitter: EventEmitter, }); @@ -63,8 +64,16 @@ describe.skip('responseStreamData:', () => { assert.strictEqual(data, postBody.toString()); done(); }); - return responseStreamData(errCode, overrideHeaders, resHeaders, - dataLocations, dataRetrievalParams, response, null, log); + return responseStreamData( + errCode, + overrideHeaders, + resHeaders, + dataLocations, + dataRetrievalParams, + response, + null, + log, + ); }); it('should stream full requested object data for two part object', done => { @@ -81,7 +90,8 @@ describe.skip('responseStreamData:', () => { dataStore: 'mem', start: 11, size: 11, - }]; + }, + ]; const response = httpMocks.createResponse({ eventEmitter: EventEmitter, }); @@ -91,17 +101,27 @@ describe.skip('responseStreamData:', () => { assert.strictEqual(data, doublePostBody); done(); }); - return responseStreamData(errCode, overrideHeaders, resHeaders, - dataLocations, dataRetrievalParams, response, null, log); + return responseStreamData( + errCode, + overrideHeaders, + resHeaders, + dataLocations, + dataRetrievalParams, + response, + null, + log, + ); }); it('#334 non-regression test, destroy connection on error', done => { - const dataLocations = [{ - key: 1, - dataStore: 'mem', - start: 0, - size: 11, - }]; + const dataLocations = [ + { + key: 1, + dataStore: 'mem', + start: 0, + size: 11, + }, + ]; const prev = data.get; data.get = (objectGetInfo, response, log, cb) => { setTimeout(() => cb(errors.InternalError), 1000); @@ -117,12 +137,19 @@ describe.skip('responseStreamData:', () => { response.on('end', () => { data.get = prev; if (!destroyed) { - return done(new Error('end reached instead of destroying ' + - 'connection')); + return done(new Error('end reached instead of destroying ' + 'connection')); } return done(); }); - return responseStreamData(errCode, overrideHeaders, resHeaders, - dataLocations, dataRetrievalParams, response, null, log); + return responseStreamData( + errCode, + overrideHeaders, + resHeaders, + dataLocations, + dataRetrievalParams, + response, + null, + log, + ); }); }); diff --git a/tests/unit/utils/serverAccessLogger.js b/tests/unit/utils/serverAccessLogger.js index 13cc1ae68e..648b204d6a 100644 --- a/tests/unit/utils/serverAccessLogger.js +++ b/tests/unit/utils/serverAccessLogger.js @@ -477,7 +477,7 @@ describe('serverAccessLogger utility functions', () => { it('should return Content-Length from response for objectGet', () => { const request = { apiMethod: 'objectGet' }; const response = { - getHeader: name => name === 'Content-Length' ? '12345' : null, + getHeader: name => (name === 'Content-Length' ? '12345' : null), }; const result = getObjectSize(request, response); assert.strictEqual(result, 12345); @@ -510,7 +510,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length of 0 for objectGet', () => { const request = { apiMethod: 'objectGet' }; const response = { - getHeader: name => name === 'Content-Length' ? '0' : null, + getHeader: name => (name === 'Content-Length' ? '0' : null), }; const result = getObjectSize(request, response); assert.strictEqual(result, 0); @@ -519,7 +519,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length of number 0 for objectGet', () => { const request = { apiMethod: 'objectGet' }; const response = { - getHeader: name => name === 'Content-Length' ? 0 : null, + getHeader: name => (name === 'Content-Length' ? 0 : null), }; const result = getObjectSize(request, response); assert.strictEqual(result, 0); @@ -604,7 +604,7 @@ describe('serverAccessLogger utility functions', () => { it('should return Content-Length from response when bytesSent is not provided', () => { const res = { - getHeader: name => name === 'Content-Length' ? '67890' : null, + getHeader: name => (name === 'Content-Length' ? '67890' : null), }; const result = getBytesSent(res, null); assert.strictEqual(result, '67890'); @@ -625,7 +625,7 @@ describe('serverAccessLogger utility functions', () => { it('should prefer bytesSent over Content-Length', () => { const res = { - getHeader: name => name === 'Content-Length' ? '99999' : null, + getHeader: name => (name === 'Content-Length' ? '99999' : null), }; const bytesSent = 11111; const result = getBytesSent(res, bytesSent); @@ -634,7 +634,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle bytesSent as 0', () => { const res = { - getHeader: name => name === 'Content-Length' ? '99999' : null, + getHeader: name => (name === 'Content-Length' ? '99999' : null), }; const bytesSent = 0; const result = getBytesSent(res, bytesSent); @@ -643,7 +643,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length as number 0', () => { const res = { - getHeader: name => name === 'Content-Length' ? 0 : null, + getHeader: name => (name === 'Content-Length' ? 0 : null), }; const result = getBytesSent(res, null); assert.strictEqual(result, 0); @@ -651,7 +651,7 @@ describe('serverAccessLogger utility functions', () => { it('should handle Content-Length as string "0"', () => { const res = { - getHeader: name => name === 'Content-Length' ? '0' : null, + getHeader: name => (name === 'Content-Length' ? '0' : null), }; const result = getBytesSent(res, null); assert.strictEqual(result, '0'); @@ -893,10 +893,10 @@ describe('serverAccessLogger utility functions', () => { analyticsAccountName: 'testAccount', analyticsUserName: 'testUser', analyticsBytesDeleted: 0, - startTime: 1000000000n, - onFinishEndTime: 1009000000n, + startTime: 1000000000n, + onFinishEndTime: 1009000000n, startTurnAroundTime: 1003000000n, - onCloseEndTime: 1020500000n, + onCloseEndTime: 1020500000n, startTimeUnixMS: 1234567890000, bucketOwner: 'bucketOwner123', bucketName: 'test-bucket', @@ -915,8 +915,8 @@ describe('serverAccessLogger utility functions', () => { headers: { 'content-length': '1024', 'user-agent': 'aws-cli/2.0.0', - 'referer': 'https://example.com', - 'host': 's3.amazonaws.com', + referer: 'https://example.com', + host: 's3.amazonaws.com', 'x-forwarded-for': '192.168.1.100', }, parsedContentLength: 1024, @@ -938,7 +938,9 @@ describe('serverAccessLogger utility functions', () => { }, statusCode: 200, getHeader: name => { - if (name === 'Content-Length') {return '2048';} + if (name === 'Content-Length') { + return '2048'; + } return null; }, }; @@ -1213,7 +1215,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const writtenData = mockLogger.write.firstCall.args[0]; assert.strictEqual(writtenData.endsWith('\n'), true); - + // Should be valid JSON without the newline const jsonData = writtenData.trim(); assert.doesNotThrow(() => JSON.parse(jsonData)); @@ -1437,7 +1439,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const loggedData = JSON.parse(mockLogger.write.firstCall.args[0].trim()); - + // Verify operation is REST.GET.BACKBEAT assert.strictEqual(loggedData.operation, 'REST.GET.BACKBEAT'); }); @@ -1467,7 +1469,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(mockLogger.write.callCount, 1); const loggedData = JSON.parse(mockLogger.write.firstCall.args[0].trim()); - + // Verify loggingEnabled is false (overridden by backbeat) assert.strictEqual(loggedData.loggingEnabled, false); // But TargetBucket and TargetPrefix should still be logged @@ -1760,8 +1762,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -1832,13 +1833,14 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.objectKey, 'replicated.txt'); // Requester is the assumed-role ARN from auth - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); // requestURI is synthesized to look like a normal S3 PUT, // not the internal /_/backbeat/data path - assert.strictEqual(loggedData.requestURI, - 'PUT /dest-bucket/replicated.txt HTTP/1.1'); + assert.strictEqual(loggedData.requestURI, 'PUT /dest-bucket/replicated.txt HTTP/1.1'); // HTTP-layer fields that AWS blanks for replication assert.strictEqual(loggedData.clientIP, undefined); @@ -1861,8 +1863,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -1910,8 +1911,7 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.DELETE.OBJECT'); assert.strictEqual(loggedData.loggingEnabled, true); - assert.strictEqual(loggedData.requestURI, - 'DELETE /dest-bucket/replicated.txt HTTP/1.1'); + assert.strictEqual(loggedData.requestURI, 'DELETE /dest-bucket/replicated.txt HTTP/1.1'); // HTTP-layer fields that AWS blanks for replication assert.strictEqual(loggedData.clientIP, undefined); @@ -1919,8 +1919,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.referer, undefined); // Replication identity preserved - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); }); it('should produce a REST.PUT.OBJECT_TAGGING entry for tag-only replication', () => { @@ -1930,8 +1932,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -1981,8 +1982,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.PUT.OBJECT_TAGGING'); assert.strictEqual(loggedData.loggingEnabled, true); - assert.strictEqual(loggedData.requestURI, - `PUT /dest-bucket/replicated.txt?tagging&versionId=${versionId} HTTP/1.1`); + assert.strictEqual( + loggedData.requestURI, + `PUT /dest-bucket/replicated.txt?tagging&versionId=${versionId} HTTP/1.1`, + ); assert.strictEqual(loggedData.versionId, versionId); // HTTP-layer fields that AWS blanks for replication @@ -1991,8 +1994,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.referer, undefined); // Replication identity preserved - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); }); it('should produce a REST.PUT.ACL entry for ACL-only replication', () => { @@ -2002,8 +2007,7 @@ describe('serverAccessLogger utility functions', () => { getCanonicalID: () => 'replication-canonical-id', isRequesterPublicUser: () => false, isRequesterAnIAMUser: () => false, - getArn: () => - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + getArn: () => 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', getAuthVersion: () => 'AWS4-HMAC-SHA256', getAuthType: () => 'REST-HEADER', getAccessKey: () => 'replication-access-key', @@ -2054,8 +2058,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.operation, 'REST.PUT.ACL'); assert.strictEqual(loggedData.loggingEnabled, true); - assert.strictEqual(loggedData.requestURI, - `PUT /dest-bucket/replicated.txt?acl&versionId=${versionId} HTTP/1.1`); + assert.strictEqual( + loggedData.requestURI, + `PUT /dest-bucket/replicated.txt?acl&versionId=${versionId} HTTP/1.1`, + ); assert.strictEqual(loggedData.versionId, versionId); assert.strictEqual(loggedData.aclRequired, 'Yes'); @@ -2065,8 +2071,10 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.referer, undefined); // Replication identity preserved - assert.strictEqual(loggedData.requester, - 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication'); + assert.strictEqual( + loggedData.requester, + 'arn:aws:sts::123456789012:assumed-role/replication-role/backbeat-replication', + ); }); it('should log replication with error code on failure', () => { @@ -2109,7 +2117,5 @@ describe('serverAccessLogger utility functions', () => { assert.strictEqual(loggedData.errorCode, 'InternalError'); assert.strictEqual(loggedData.httpCode, 500); }); - }); }); - diff --git a/tests/unit/utils/setPartRanges.js b/tests/unit/utils/setPartRanges.js index 52c261e4e1..ef56b3e1e1 100644 --- a/tests/unit/utils/setPartRanges.js +++ b/tests/unit/utils/setPartRanges.js @@ -6,76 +6,85 @@ describe('setPartRanges function', () => { it('should set range on a one part object', () => { const dataLocations = [{ key: '1' }]; const outerRange = [2, 8]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ - key: '1', - range: [2, 8], - }]); + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { + key: '1', + range: [2, 8], + }, + ]); }); - it('for a 3-part object, should include full first part, set range on ' + - 'middle part and exclude last part if range request starts at 0' + - 'and ends in the middle of the second part', + it( + 'for a 3-part object, should include full first part, set range on ' + + 'middle part and exclude last part if range request starts at 0' + + 'and ends in the middle of the second part', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [0, 10]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '1', size: '4', start: '0' }, - { key: '2', size: '7', start: '4', range: [0, 6] }]); - }); + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { key: '1', size: '4', start: '0' }, + { key: '2', size: '7', start: '4', range: [0, 6] }, + ]); + }, + ); - it('for a 3-part object, should include part of first part, all of ' + - 'second part and part of third part if range request starts within ' + - 'first part and ends before end of last part', + it( + 'for a 3-part object, should include part of first part, all of ' + + 'second part and part of third part if range request starts within ' + + 'first part and ends before end of last part', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [2, 18]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '1', size: '2', start: '0', - range: [2, 3] }, - { key: '2', size: '10', start: '4' }, - { key: '3', size: '5', start: '14', range: [0, 4] }, + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { key: '1', size: '2', start: '0', range: [2, 3] }, + { key: '2', size: '10', start: '4' }, + { key: '3', size: '5', start: '14', range: [0, 4] }, ]); - }); + }, + ); - it('for a 3-part object, should include only a range of the middle part ' + - 'if the range excludes both the beginning and the end', + it( + 'for a 3-part object, should include only a range of the middle part ' + + 'if the range excludes both the beginning and the end', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [5, 7]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '2', size: '3', start: '4', - range: [1, 3] }, - ]); - }); + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [{ key: '2', size: '3', start: '4', range: [1, 3] }]); + }, + ); - it('for a 3-part object, should include only a range of the middle part ' + - 'and all of the third part if the range excludes a portion of the ' + - 'beginning', + it( + 'for a 3-part object, should include only a range of the middle part ' + + 'and all of the third part if the range excludes a portion of the ' + + 'beginning', () => { - const dataLocations = [{ key: '1', size: '4', start: '0' }, + const dataLocations = [ + { key: '1', size: '4', start: '0' }, { key: '2', size: '10', start: '4' }, { key: '3', size: '20', start: '14' }, ]; const outerRange = [5, 34]; - const actual = - setPartRanges(dataLocations, outerRange); - assert.deepStrictEqual(actual, [{ key: '2', size: '9', start: '4', - range: [1, 9] }, + const actual = setPartRanges(dataLocations, outerRange); + assert.deepStrictEqual(actual, [ + { key: '2', size: '9', start: '4', range: [1, 9] }, { key: '3', size: '20', start: '14' }, ]); - }); + }, + ); }); diff --git a/tests/unit/utils/validateSearch.js b/tests/unit/utils/validateSearch.js index 4b803278aa..47dc0a1485 100644 --- a/tests/unit/utils/validateSearch.js +++ b/tests/unit/utils/validateSearch.js @@ -1,8 +1,6 @@ const assert = require('assert'); const { errorInstances } = require('arsenal'); -const validateSearch = - require('../../../lib/api/apiUtils/bucket/validateSearch'); - +const validateSearch = require('../../../lib/api/apiUtils/bucket/validateSearch'); describe('validate search where clause', () => { const tests = [ @@ -12,33 +10,30 @@ describe('validate search where clause', () => { result: undefined, }, { - it: 'should allow a simple search with known ' + - 'column attribute', + it: 'should allow a simple search with known ' + 'column attribute', searchParams: '`content-length`="10"', result: undefined, }, { it: 'should allow valid search with AND', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'AND `x-amz-meta-age`="5"', + searchParams: '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5"', result: undefined, }, { it: 'should allow valid search with OR', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'OR `x-amz-meta-age`="5"', + searchParams: '`x-amz-meta-dog`="labrador" ' + 'OR `x-amz-meta-age`="5"', result: undefined, }, { it: 'should allow valid search with double AND', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'AND `x-amz-meta-age`="5" ' + - 'AND `x-amz-meta-whatever`="ok"', + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'AND `x-amz-meta-whatever`="ok"', result: undefined, }, { it: 'should allow valid chained search with tables and columns', - searchParams: '`x-amz-meta-dog`="labrador" ' + + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'AND `content-length`="10"' + 'OR isDeleteMarker="true"' + @@ -47,70 +42,70 @@ describe('validate search where clause', () => { }, { it: 'should allow valid LIKE search', - searchParams: '`x-amz-meta-dog` LIKE "lab%" ' + - 'AND `x-amz-meta-age` LIKE "5%" ' + - 'AND `content-length`="10"', + searchParams: + '`x-amz-meta-dog` LIKE "lab%" ' + 'AND `x-amz-meta-age` LIKE "5%" ' + 'AND `content-length`="10"', result: undefined, }, { it: 'should disallow a LIKE search with invalid attribute', searchParams: '`x-zma-meta-dog` LIKE "labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { it: 'should disallow a simple search with unknown attribute', searchParams: '`x-zma-meta-dog`="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { - it: 'should disallow a compound search with unknown ' + - 'attribute on right', - searchParams: '`x-amz-meta-dog`="labrador" AND ' + - '`x-zma-meta-dog`="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + it: 'should disallow a compound search with unknown ' + 'attribute on right', + searchParams: '`x-amz-meta-dog`="labrador" AND ' + '`x-zma-meta-dog`="labrador"', + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { - it: 'should disallow a compound search with unknown ' + - 'attribute on left', - searchParams: '`x-zma-meta-dog`="labrador" AND ' + - '`x-amz-meta-dog`="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-dog'), + it: 'should disallow a compound search with unknown ' + 'attribute on left', + searchParams: '`x-zma-meta-dog`="labrador" AND ' + '`x-amz-meta-dog`="labrador"', + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-dog', + ), }, { - it: 'should disallow a chained search with one invalid ' + - 'table attribute', - searchParams: '`x-amz-meta-dog`="labrador" ' + - 'AND `x-amz-meta-age`="5" ' + - 'OR `x-zma-meta-whatever`="ok"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: x-zma-meta-whatever'), + it: 'should disallow a chained search with one invalid ' + 'table attribute', + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'OR `x-zma-meta-whatever`="ok"', + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: x-zma-meta-whatever', + ), }, { - it: 'should disallow a simple search with unknown ' + - 'column attribute', + it: 'should disallow a simple search with unknown ' + 'column attribute', searchParams: 'whatever="labrador"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: whatever'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: whatever', + ), }, { - it: 'should disallow a chained search with one invalid ' + - 'column attribute', - searchParams: '`x-amz-meta-dog`="labrador" ' + + it: 'should disallow a chained search with one invalid ' + 'column attribute', + searchParams: + '`x-amz-meta-dog`="labrador" ' + 'AND `x-amz-meta-age`="5" ' + 'OR madeUp="something"' + 'OR `x-amz-meta-whatever`="ok"', - result: errorInstances.InvalidArgument.customizeDescription('Search ' + - 'param contains unknown attribute: madeUp'), + result: errorInstances.InvalidArgument.customizeDescription( + 'Search ' + 'param contains unknown attribute: madeUp', + ), }, { it: 'should disallow unsupported query operators', searchParams: 'x-amz-meta-dog BETWEEN "labrador"', result: errorInstances.InvalidArgument.customizeDescription( - 'Invalid sql where clause sent as search query'), + 'Invalid sql where clause sent as search query', + ), }, { it: 'should allow a simple search with tag query', @@ -126,8 +121,7 @@ describe('validate search where clause', () => { tests.forEach(test => { it(test.it, () => { - const actualResult = - validateSearch(test.searchParams); + const actualResult = validateSearch(test.searchParams); if (test.result === undefined) { assert(typeof actualResult.ast === 'object'); } else { diff --git a/tests/utapi/awsNodeSdk.js b/tests/utapi/awsNodeSdk.js index c0d1030755..975f3cc6a2 100644 --- a/tests/utapi/awsNodeSdk.js +++ b/tests/utapi/awsNodeSdk.js @@ -26,13 +26,15 @@ function wait(timeoutMs, cb) { } function createBucket(bucket, cb) { - s3Client.send(new CreateBucketCommand({ Bucket: bucket })) + s3Client + .send(new CreateBucketCommand({ Bucket: bucket })) .then(data => cb(null, data)) .catch(cb); } function deleteBucket(bucket, cb) { - s3Client.send(new DeleteBucketCommand({ Bucket: bucket })) + s3Client + .send(new DeleteBucketCommand({ Bucket: bucket })) .then(() => cb(null)) .catch(cb); } @@ -44,13 +46,15 @@ function putObject(bucket, key, size, cb) { Key: key, Body: body, }; - s3Client.send(new PutObjectCommand(params)) + s3Client + .send(new PutObjectCommand(params)) .then(data => cb(null, data)) .catch(cb); } function deleteObject(bucket, key, cb) { - s3Client.send(new DeleteObjectCommand({ Bucket: bucket, Key: key })) + s3Client + .send(new DeleteObjectCommand({ Bucket: bucket, Key: key })) .then(() => cb(null)) .catch(cb); } @@ -62,14 +66,16 @@ function deleteObjects(bucket, keys, cb) { Bucket: bucket, Delete: deleteRequest, }; - s3Client.send(new DeleteObjectsCommand(params)) + s3Client + .send(new DeleteObjectsCommand(params)) .then(() => cb(null)) .catch(cb); } function copyObject(bucket, key, cb) { const params = { Bucket: bucket, CopySource: `${bucket}/${key}`, Key: `${key}-copy` }; - s3Client.send(new CopyObjectCommand(params)) + s3Client + .send(new CopyObjectCommand(params)) .then(() => cb(null)) .catch(cb); } @@ -77,7 +83,8 @@ function copyObject(bucket, key, cb) { function enableVersioning(bucket, enable, cb) { const versioningStatus = { Status: enable ? 'Enabled' : 'Disabled' }; const params = { Bucket: bucket, VersioningConfiguration: versioningStatus }; - s3Client.send(new PutBucketVersioningCommand(params)) + s3Client + .send(new PutBucketVersioningCommand(params)) .then(() => cb(null)) .catch(cb); } @@ -101,24 +108,24 @@ async function removeAllVersions(params, callback) { try { const bucket = params.Bucket; const data = await s3Client.send(new ListObjectVersionsCommand(params)); - + if (data.DeleteMarkers && data.DeleteMarkers.length > 0) { await deleteVersionList(data.DeleteMarkers, bucket); } - + if (data.Versions && data.Versions.length > 0) { await deleteVersionList(data.Versions, bucket); } - + if (data.IsTruncated) { - const nextParams = { - Bucket: bucket, - KeyMarker: data.NextKeyMarker, - VersionIdMarker: data.NextVersionIdMarker + const nextParams = { + Bucket: bucket, + KeyMarker: data.NextKeyMarker, + VersionIdMarker: data.NextVersionIdMarker, }; await removeAllVersions(nextParams); } - + callback(); } catch (error) { callback(error); @@ -130,46 +137,58 @@ function objectMPU(bucket, key, parts, partSize, callback) { let uploadId = null; const partNumbers = Array.from(Array(parts).keys()); const initiateMPUParams = { Bucket: bucket, Key: key }; - async.waterfall([ - next => s3Client.send(new CreateMultipartUploadCommand(initiateMPUParams)) - .then(data => { - uploadId = data.UploadId; - return next(); - }) - .catch(next), - next => - async.mapLimit(partNumbers, 1, (partNumber, callback) => { - const body = Buffer.alloc(partSize); - const uploadPartParams = { + async.waterfall( + [ + next => + s3Client + .send(new CreateMultipartUploadCommand(initiateMPUParams)) + .then(data => { + uploadId = data.UploadId; + return next(); + }) + .catch(next), + next => + async.mapLimit( + partNumbers, + 1, + (partNumber, callback) => { + const body = Buffer.alloc(partSize); + const uploadPartParams = { + Bucket: bucket, + Key: key, + PartNumber: partNumber + 1, + UploadId: uploadId, + Body: body, + }; + s3Client + .send(new UploadPartCommand(uploadPartParams)) + .then(data => callback(null, data.ETag)) + .catch(callback); + }, + (err, results) => { + if (err) { + return next(err); + } + ETags = results; + return next(); + }, + ), + next => { + const completeRequest = { Parts: partNumbers.map(n => ({ ETag: ETags[n], PartNumber: n + 1 })) }; + const params = { Bucket: bucket, Key: key, - PartNumber: partNumber + 1, + MultipartUpload: completeRequest, UploadId: uploadId, - Body: body, }; - s3Client.send(new UploadPartCommand(uploadPartParams)) - .then(data => callback(null, data.ETag)) - .catch(callback); - }, (err, results) => { - if (err) { - return next(err); - } - ETags = results; - return next(); - }), - next => { - const completeRequest = { Parts: partNumbers.map(n => ({ ETag: ETags[n], PartNumber: n + 1 })) }; - const params = { - Bucket: bucket, - Key: key, - MultipartUpload: completeRequest, - UploadId: uploadId, - }; - s3Client.send(new CompleteMultipartUploadCommand(params)) - .then(data => next(null, data)) - .catch(next); - }, - ], callback); + s3Client + .send(new CompleteMultipartUploadCommand(params)) + .then(data => next(null, data)) + .catch(next); + }, + ], + callback, + ); } function removeVersions(buckets, cb) { @@ -177,7 +196,8 @@ function removeVersions(buckets, cb) { } function getObject(bucket, key, cb) { - s3Client.send(new GetObjectCommand({ Bucket: bucket, Key: key })) + s3Client + .send(new GetObjectCommand({ Bucket: bucket, Key: key })) .then(data => cb(null, data)) .catch(cb); } @@ -195,7 +215,7 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { } before(() => { - s3Client = new S3Client(getConfig('default')); + s3Client = new S3Client(getConfig('default')); utapi.start(); }); afterEach(() => { @@ -206,18 +226,23 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { }); it('should set metrics for createBucket and deleteBucket', done => { const bucket = 'bucket1'; - async.series([ - next => createBucket(bucket, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + ], + done, + ); }); it('should set metrics for putObject and deleteObject', done => { const bucket = 'bucket2'; @@ -226,75 +251,93 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { const obj2Size = objectSize * 2; const key1 = '1.txt'; const key2 = '2.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key1, obj1Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size, 0, 1); - next(); - }), - next => putObject(bucket, key2, obj2Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 2); - next(); - }), - next => deleteObject(bucket, key1, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj2Size, 0, 1); - next(); - }), - next => deleteObject(bucket, key2, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key1, obj1Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size, 0, 1); + next(); + }), + next => putObject(bucket, key2, obj2Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 2); + next(); + }), + next => deleteObject(bucket, key1, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj2Size, 0, 1); + next(); + }), + next => deleteObject(bucket, key2, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for copyObject', done => { const bucket = 'bucket3'; const objectSize = 1024 * 1024 * 2; const key = '3.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key, objectSize, next), - next => copyObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize * 2, 0, 2); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize, 0, 1); - next(); - }), - next => deleteObject(bucket, `${key}-copy`, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key, objectSize, next), + next => copyObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize * 2, 0, 2); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize, 0, 1); + next(); + }), + next => deleteObject(bucket, `${key}-copy`, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for getObject', done => { const bucket = 'bucket4'; const objectSize = 1024 * 1024 * 2; const key = '4.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key, objectSize, next), - next => getObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize, objectSize, 1); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, objectSize, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key, objectSize, next), + next => getObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize, objectSize, 1); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, objectSize, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiObjectDelete', done => { const bucket = 'bucket5'; @@ -303,133 +346,164 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { const obj2Size = objectSize * 1; const key1 = '1.txt'; const key2 = '2.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key1, obj1Size, next), - next => wait(WAIT_MS, next), - next => putObject(bucket, key2, obj2Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 2); - next(); - }), - next => deleteObjects(bucket, [key1, key2], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key1, obj1Size, next), + next => wait(WAIT_MS, next), + next => putObject(bucket, key2, obj2Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 2); + next(); + }), + next => deleteObjects(bucket, [key1, key2], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiPartUpload', done => { const bucket = 'bucket6'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '6.txt'; - async.series([ - next => createBucket(bucket, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts, 0, 1); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts, 0, 1); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics in versioned bucket', done => { const bucket = 'bucket7'; const objectSize = 1024 * 1024; const key = '7.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => putObject(bucket, key, objectSize, next), - next => wait(WAIT_MS, next), - next => putObject(bucket, key, objectSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize * 2, 0, 2); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(objectSize * 2, 0, 3); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => putObject(bucket, key, objectSize, next), + next => wait(WAIT_MS, next), + next => putObject(bucket, key, objectSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize * 2, 0, 2); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(objectSize * 2, 0, 3); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multipartUpload in a versioned bucket', done => { const bucket = 'bucket8'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '8.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts, 0, 1); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts, 0, 1); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multipartUpload overwrite in a versioned bucket', done => { const bucket = 'bucket9'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '9.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts * 2, 0, 2); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts * 2, 0, 2); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiPartUpload overwrite', done => { const bucket = 'bucket10'; const partSize = 1024 * 1024 * 6; const parts = 2; const key = '10.txt'; - async.series([ - next => createBucket(bucket, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => objectMPU(bucket, key, parts, partSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(partSize * parts, 0, 1); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => objectMPU(bucket, key, parts, partSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(partSize * parts, 0, 1); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should set metrics for multiObjectDelete in a versioned bucket', done => { const bucket = 'bucket11'; @@ -438,47 +512,58 @@ describe('utapi v2 metrics incoming and outgoing bytes', function t() { const obj2Size = objectSize * 1; const key1 = '1.txt'; const key2 = '2.txt'; - async.series([ - next => createBucket(bucket, next), - next => enableVersioning(bucket, true, next), - next => putObject(bucket, key1, obj1Size, next), - next => wait(WAIT_MS, next), - next => putObject(bucket, key2, obj2Size, next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 2); - next(); - }), - next => deleteObjects(bucket, [key1, key2], next), - next => wait(WAIT_MS, () => { - checkMetrics(obj1Size + obj2Size, 0, 4); - next(); - }), - next => removeVersions([bucket], next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => enableVersioning(bucket, true, next), + next => putObject(bucket, key1, obj1Size, next), + next => wait(WAIT_MS, next), + next => putObject(bucket, key2, obj2Size, next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 2); + next(); + }), + next => deleteObjects(bucket, [key1, key2], next), + next => + wait(WAIT_MS, () => { + checkMetrics(obj1Size + obj2Size, 0, 4); + next(); + }), + next => removeVersions([bucket], next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); it('should not push a metric for a filtered bucket', done => { const bucket = 'utapi-event-filter-deny-bucket'; const objSize = 2 * 1024 * 1024; const key = '1.txt'; - async.series([ - next => createBucket(bucket, next), - next => putObject(bucket, key, objSize, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteObject(bucket, key, next), - next => wait(WAIT_MS, () => { - checkMetrics(0, 0, 0); - next(); - }), - next => deleteBucket(bucket, next), - ], done); + async.series( + [ + next => createBucket(bucket, next), + next => putObject(bucket, key, objSize, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteObject(bucket, key, next), + next => + wait(WAIT_MS, () => { + checkMetrics(0, 0, 0); + next(); + }), + next => deleteBucket(bucket, next), + ], + done, + ); }); }); diff --git a/tests/utapi/utilities.js b/tests/utapi/utilities.js index b8b7295f9c..5046e029b9 100644 --- a/tests/utapi/utilities.js +++ b/tests/utapi/utilities.js @@ -4,481 +4,520 @@ const werelogs = require('werelogs'); const _config = require('../../lib/Config').config; const { makeAuthInfo } = require('../unit/helpers'); -const testEvents = [{ - action: 'getObject', - metrics: { - bucket: 'bucket1', - keys: ['1.txt'], - newByteLength: 2, - oldByteLength: null, - versionId: undefined, - location: 'us-west-1', - numberOfObjects: 1, - byteLength: null, - isDelete: false, - }, - expected: { - objectDelta: 1, - sizeDelta: 0, - incomingBytes: 0, - outgoingBytes: 2, - }, -}, { - action: 'deleteObject', - metrics: { - bucket: 'bucket1', - keys: ['1.txt'], - byteLength: 2, - numberOfObjects: 1, - location: 'us-west-1', - isDelete: true, - }, - expected: { - objectDelta: -1, - sizeDelta: -2, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'listBucket', - metrics: { - bucket: 'bucket1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putObject', - metrics: { - bucket: 'bucket1', - keys: ['2.txt'], - newByteLength: 2, - oldByteLength: null, - versionId: undefined, - location: 'us-west-1', - numberOfObjects: 1, - }, - expected: { - objectDelta: 1, - sizeDelta: 2, - incomingBytes: 2, - outgoingBytes: 0, - }, -}, { - action: 'listBucket', - metrics: { - bucket: 'bucket1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'headObject', - metrics: { - bucket: 'bucket1', - keys: ['1.txt'], - versionId: undefined, - location: 'us-west-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'abortMultipartUpload', - metrics: { - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - byteLength: 26, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: -26, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'completeMultipartUpload', - metrics: { - oldByteLength: null, - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - versionId: undefined, - numberOfObjects: 1, - location: 'us-east-1', - }, - expected: { - objectDelta: 1, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'createBucket', - metrics: { - bucket: 'deletebucketpolicy-test-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucket', - metrics: { - bucket: 'deletebucketpolicy-test-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucketCors', - metrics: { - bucket: 'testdeletecorsbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucketReplication', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'deleteBucketWebsite', - metrics: { - bucket: 'testdeletewebsitebucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketAcl', - metrics: { - bucket: 'putbucketaclfttest', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketCors', - metrics: { - bucket: 'testgetcorsbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketLocation', - metrics: { - bucket: 'testgetlocationbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketNotification', - metrics: { - bucket: 'notificationtestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketObjectLock', - metrics: { - bucket: 'mock-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketReplication', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketVersioning', - metrics: { - bucket: 'bucket-with-object-lock', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getBucketWebsite', - metrics: { - bucket: 'testgetwebsitetestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'getObjectTagging', - metrics: { - bucket: 'completempu1615102906771', - keys: ['keywithtags'], - versionId: undefined, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'headObject', - metrics: { - bucket: 'supersourcebucket81033016532', - keys: ['supersourceobject'], - versionId: undefined, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'initiateMultipartUpload', - metrics: { - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'listMultipartUploadParts', - metrics: { - bucket: 'ftest-mybucket-74', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'multiObjectDelete', - metrics: { - bucket: 'completempu1615102906771', - keys: [undefined], - byteLength: 3, - numberOfObjects: 1, - removedDeleteMarkers: 1, - isDelete: true, - }, - expected: { - objectDelta: -2, - sizeDelta: -3, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketAcl', - metrics: { - bucket: 'putbucketaclfttest', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketCors', - metrics: { - bucket: 'testcorsbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketNotification', - metrics: { - bucket: 'notificationtestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketObjectLock', - metrics: { - bucket: 'mock-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketReplication', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketVersioning', - metrics: { - bucket: 'source-bucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'putBucketWebsite', - metrics: { - bucket: 'testgetwebsitetestbucket', - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'uploadPart', - metrics: { - bucket: 'ftest-mybucket-74', - keys: ['toAbort&<>"\''], - newByteLength: 5242880, - oldByteLength: null, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: 5242880, - incomingBytes: 5242880, - outgoingBytes: 0, - }, -}, { - action: 'uploadPartCopy', - metrics: { - bucket: 'destinationbucket815502017', - keys: ['copycatobject'], - newByteLength: 26, - oldByteLength: null, - location: 'us-east-1', - }, - expected: { - objectDelta: undefined, - sizeDelta: 26, - incomingBytes: 26, - outgoingBytes: 0, - }, -}, { - action: 'replicateObject', - metrics: { - bucket: 'source-bucket', - keys: ['mykey'], - newByteLength: 26, - oldByteLength: null, - }, - expected: { - objectDelta: 1, - sizeDelta: 26, - incomingBytes: 26, - outgoingBytes: 0, - }, -}, { - action: 'replicateDelete', - metrics: { - bucket: 'source-bucket', - keys: ['mykey'], - }, - expected: { - objectDelta: 1, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}, { - action: 'replicateTags', - metrics: { - bucket: 'source-bucket', - keys: ['mykey'], - }, - expected: { - objectDelta: undefined, - sizeDelta: undefined, - incomingBytes: undefined, - outgoingBytes: 0, - }, -}]; +const testEvents = [ + { + action: 'getObject', + metrics: { + bucket: 'bucket1', + keys: ['1.txt'], + newByteLength: 2, + oldByteLength: null, + versionId: undefined, + location: 'us-west-1', + numberOfObjects: 1, + byteLength: null, + isDelete: false, + }, + expected: { + objectDelta: 1, + sizeDelta: 0, + incomingBytes: 0, + outgoingBytes: 2, + }, + }, + { + action: 'deleteObject', + metrics: { + bucket: 'bucket1', + keys: ['1.txt'], + byteLength: 2, + numberOfObjects: 1, + location: 'us-west-1', + isDelete: true, + }, + expected: { + objectDelta: -1, + sizeDelta: -2, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'listBucket', + metrics: { + bucket: 'bucket1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putObject', + metrics: { + bucket: 'bucket1', + keys: ['2.txt'], + newByteLength: 2, + oldByteLength: null, + versionId: undefined, + location: 'us-west-1', + numberOfObjects: 1, + }, + expected: { + objectDelta: 1, + sizeDelta: 2, + incomingBytes: 2, + outgoingBytes: 0, + }, + }, + { + action: 'listBucket', + metrics: { + bucket: 'bucket1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'headObject', + metrics: { + bucket: 'bucket1', + keys: ['1.txt'], + versionId: undefined, + location: 'us-west-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'abortMultipartUpload', + metrics: { + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + byteLength: 26, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: -26, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'completeMultipartUpload', + metrics: { + oldByteLength: null, + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + versionId: undefined, + numberOfObjects: 1, + location: 'us-east-1', + }, + expected: { + objectDelta: 1, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'createBucket', + metrics: { + bucket: 'deletebucketpolicy-test-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucket', + metrics: { + bucket: 'deletebucketpolicy-test-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucketCors', + metrics: { + bucket: 'testdeletecorsbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucketReplication', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'deleteBucketWebsite', + metrics: { + bucket: 'testdeletewebsitebucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketAcl', + metrics: { + bucket: 'putbucketaclfttest', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketCors', + metrics: { + bucket: 'testgetcorsbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketLocation', + metrics: { + bucket: 'testgetlocationbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketNotification', + metrics: { + bucket: 'notificationtestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketObjectLock', + metrics: { + bucket: 'mock-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketReplication', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketVersioning', + metrics: { + bucket: 'bucket-with-object-lock', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getBucketWebsite', + metrics: { + bucket: 'testgetwebsitetestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'getObjectTagging', + metrics: { + bucket: 'completempu1615102906771', + keys: ['keywithtags'], + versionId: undefined, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'headObject', + metrics: { + bucket: 'supersourcebucket81033016532', + keys: ['supersourceobject'], + versionId: undefined, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'initiateMultipartUpload', + metrics: { + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'listMultipartUploadParts', + metrics: { + bucket: 'ftest-mybucket-74', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'multiObjectDelete', + metrics: { + bucket: 'completempu1615102906771', + keys: [undefined], + byteLength: 3, + numberOfObjects: 1, + removedDeleteMarkers: 1, + isDelete: true, + }, + expected: { + objectDelta: -2, + sizeDelta: -3, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketAcl', + metrics: { + bucket: 'putbucketaclfttest', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketCors', + metrics: { + bucket: 'testcorsbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketNotification', + metrics: { + bucket: 'notificationtestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketObjectLock', + metrics: { + bucket: 'mock-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketReplication', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketVersioning', + metrics: { + bucket: 'source-bucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'putBucketWebsite', + metrics: { + bucket: 'testgetwebsitetestbucket', + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'uploadPart', + metrics: { + bucket: 'ftest-mybucket-74', + keys: ['toAbort&<>"\''], + newByteLength: 5242880, + oldByteLength: null, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: 5242880, + incomingBytes: 5242880, + outgoingBytes: 0, + }, + }, + { + action: 'uploadPartCopy', + metrics: { + bucket: 'destinationbucket815502017', + keys: ['copycatobject'], + newByteLength: 26, + oldByteLength: null, + location: 'us-east-1', + }, + expected: { + objectDelta: undefined, + sizeDelta: 26, + incomingBytes: 26, + outgoingBytes: 0, + }, + }, + { + action: 'replicateObject', + metrics: { + bucket: 'source-bucket', + keys: ['mykey'], + newByteLength: 26, + oldByteLength: null, + }, + expected: { + objectDelta: 1, + sizeDelta: 26, + incomingBytes: 26, + outgoingBytes: 0, + }, + }, + { + action: 'replicateDelete', + metrics: { + bucket: 'source-bucket', + keys: ['mykey'], + }, + expected: { + objectDelta: 1, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, + { + action: 'replicateTags', + metrics: { + bucket: 'source-bucket', + keys: ['mykey'], + }, + expected: { + objectDelta: undefined, + sizeDelta: undefined, + incomingBytes: undefined, + outgoingBytes: 0, + }, + }, +]; describe('utapi v2 pushmetrics utility', () => { const log = new werelogs.Logger('utapi-utility'); @@ -491,8 +530,7 @@ describe('utapi v2 pushmetrics utility', () => { before(() => { assert.strictEqual(utapiVersion, 2); - sinon.stub(UtapiClient.prototype, 'pushMetric') - .callsFake(pushMetricStub); + sinon.stub(UtapiClient.prototype, 'pushMetric').callsFake(pushMetricStub); pushMetric = require('../../lib/utapi/utilities').pushMetric; }); @@ -530,34 +568,34 @@ describe('utapi v2 pushmetrics utility', () => { ]); testEvents - .map(event => { - const modifiedEvent = event; - const authInfo = makeAuthInfo('accesskey1', 'Bart'); - authInfo.arn = `foo:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`; - modifiedEvent.metrics.authInfo = authInfo; - modifiedEvent.metrics.canonicalID = 'accesskey1'; - return modifiedEvent; - }) - .map(event => { - if (eventFilterList.has(event.action)) { - it(`should skip action ${event.action}`, () => { - _config.lifecycleRoleName = 'lifecycleTestRoleName'; - const eventPushed = pushMetric(event.action, log, event.metrics); - assert.strictEqual(eventPushed, undefined); - }); - } - return event; - }) - .forEach(event => { - if (!eventFilterList.has(event.action)) { - it(`should compute and push metrics for ${event.action}`, () => { - const eventPushed = pushMetric(event.action, log, event.metrics); - assert(eventPushed); - Object.keys(event.expected).forEach(key => { - assert.strictEqual(eventPushed[key], event.expected[key]); + .map(event => { + const modifiedEvent = event; + const authInfo = makeAuthInfo('accesskey1', 'Bart'); + authInfo.arn = `foo:assumed-role/${_config.lifecycleRoleName}/backbeat-lifecycle`; + modifiedEvent.metrics.authInfo = authInfo; + modifiedEvent.metrics.canonicalID = 'accesskey1'; + return modifiedEvent; + }) + .map(event => { + if (eventFilterList.has(event.action)) { + it(`should skip action ${event.action}`, () => { + _config.lifecycleRoleName = 'lifecycleTestRoleName'; + const eventPushed = pushMetric(event.action, log, event.metrics); + assert.strictEqual(eventPushed, undefined); }); - }); - } - }); + } + return event; + }) + .forEach(event => { + if (!eventFilterList.has(event.action)) { + it(`should compute and push metrics for ${event.action}`, () => { + const eventPushed = pushMetric(event.action, log, event.metrics); + assert(eventPushed); + Object.keys(event.expected).forEach(key => { + assert.strictEqual(eventPushed[key], event.expected[key]); + }); + }); + } + }); }); }); diff --git a/tests/utilities/bucketTagging-util.js b/tests/utilities/bucketTagging-util.js index 337780c27a..aca1d02b28 100644 --- a/tests/utilities/bucketTagging-util.js +++ b/tests/utilities/bucketTagging-util.js @@ -5,11 +5,17 @@ function assertError(err, expectedErr) { if (expectedErr === null) { assert.strictEqual(err, null, `expected no error but got '${err}'`); } else { - assert.strictEqual(err.Code, expectedErr, 'incorrect error response ' + - `code: should be '${expectedErr}' but got '${err.Code}'`); - assert.strictEqual(err.$metadata.httpStatusCode, errors[expectedErr].code, + assert.strictEqual( + err.Code, + expectedErr, + 'incorrect error response ' + `code: should be '${expectedErr}' but got '${err.Code}'`, + ); + assert.strictEqual( + err.$metadata.httpStatusCode, + errors[expectedErr].code, 'incorrect error status code: should be ' + - `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`); + `${errors[expectedErr].code}, but got '${err.$metadata.httpStatusCode}'`, + ); } } diff --git a/tests/utilities/objectLock-util.js b/tests/utilities/objectLock-util.js index 92a223e6cd..e9cae0212c 100644 --- a/tests/utilities/objectLock-util.js +++ b/tests/utilities/objectLock-util.js @@ -10,22 +10,26 @@ const versionIdUtils = versioning.VersionID; const log = new DummyRequestLogger(); function changeObjectLock(objects, newConfig, cb) { - async.each(objects, (object, next) => { - const { bucket, key, versionId } = object; - metadataGetObject(bucket, key, versionIdUtils.decode(versionId), null, log, (err, objMD) => { - assert.ifError(err); - // set newConfig as empty string to remove object lock - /* eslint-disable no-param-reassign */ - objMD.retentionMode = newConfig.mode; - objMD.retentionDate = newConfig.date; - objMD.legalHold = false; - const params = { versionId: objMD.versionId, isNull: false }; - metadata.putObjectMD(bucket, key, objMD, params, log, err => { + async.each( + objects, + (object, next) => { + const { bucket, key, versionId } = object; + metadataGetObject(bucket, key, versionIdUtils.decode(versionId), null, log, (err, objMD) => { assert.ifError(err); - next(); + // set newConfig as empty string to remove object lock + /* eslint-disable no-param-reassign */ + objMD.retentionMode = newConfig.mode; + objMD.retentionDate = newConfig.date; + objMD.legalHold = false; + const params = { versionId: objMD.versionId, isNull: false }; + metadata.putObjectMD(bucket, key, objMD, params, log, err => { + assert.ifError(err); + next(); + }); }); - }); - }, cb); + }, + cb, + ); } module.exports = changeObjectLock; diff --git a/yamllint.yml b/yamllint.yml index 1a62f98541..a47c310d44 100644 --- a/yamllint.yml +++ b/yamllint.yml @@ -1,5 +1,4 @@ --- - extends: default rules: