From fa0442f91efd07f524fffc6d4a4a8cdc315a6569 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Mon, 24 Aug 2026 09:34:14 +0200 Subject: [PATCH] Allow writing objects directly to a cold location Users had no way to store an object in a cold location without first writing it hot and waiting for a lifecycle transition rule to kick in, which is impractical when the intent is known upfront. PutObject, CreateMultipartUpload and CopyObject now accept the name of a cold location in x-amz-storage-class. The data is still written to the hot location as usual, but the object is stamped with the cold storage class and flagged as transition-in-progress, so the lifecycle queue populator picks it up from the oplog and drives the transition. No Kafka message is written by cloudserver itself, and a requeue keeps the flag. This is gated by a new off-by-default `enableDirectToCold` option (ENABLE_DIRECT_TO_COLD). Which identities may use a given storage class is left to the existing s3:x-amz-storage-class IAM condition key. Also drops the dead CLDSRV-639 lowercase/uppercase storage class handling in CreateMultipartUpload: the value is validated beforehand, so it can be stored as-is. Issue: CLDSRV-917 --- lib/Config.js | 10 + .../apiUtils/object/createAndStoreObject.js | 7 + lib/api/apiUtils/object/storageClass.js | 50 +++ lib/api/completeMultipartUpload.js | 7 + lib/api/initiateMultipartUpload.js | 21 +- lib/api/objectCopy.js | 18 +- lib/api/objectPut.js | 8 +- lib/services.js | 10 +- tests/unit/api/apiUtils/storageClass.js | 98 +++++ tests/unit/api/directToCold.js | 363 ++++++++++++++++++ 10 files changed, 565 insertions(+), 27 deletions(-) create mode 100644 lib/api/apiUtils/object/storageClass.js create mode 100644 tests/unit/api/apiUtils/storageClass.js create mode 100644 tests/unit/api/directToCold.js diff --git a/lib/Config.js b/lib/Config.js index d283300050..19fef0ebcc 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -1916,6 +1916,16 @@ class Config extends EventEmitter { this.enableVeeamRoute = config.enableVeeamRoute; } + // Allow object write APIs to name a cold location in x-amz-storage-class: data stays + // in hot storage, and the transition is triggered async from the oplog event + this.enableDirectToCold = process.env.ENABLE_DIRECT_TO_COLD !== undefined + ? process.env.ENABLE_DIRECT_TO_COLD === 'true' + : config.enableDirectToCold ?? false; + assert( + typeof this.enableDirectToCold === 'boolean', + 'bad config: enableDirectToCold must be a boolean', + ); + // Parse and validate all rate limiting configuration this.rateLimiting = parseRateLimitConfig(config.rateLimiting); diff --git a/lib/api/apiUtils/object/createAndStoreObject.js b/lib/api/apiUtils/object/createAndStoreObject.js index 82f26ccfc5..9ac02b480e 100644 --- a/lib/api/apiUtils/object/createAndStoreObject.js +++ b/lib/api/apiUtils/object/createAndStoreObject.js @@ -13,6 +13,7 @@ const getReplicationInfo = require('./getReplicationInfo'); const { config } = require('../../../Config'); const validateWebsiteHeader = require('./websiteServing').validateWebsiteHeader; const applyZenkoUserMD = require('./applyZenkoUserMD'); +const { isColdStorageClass } = require('./storageClass'); const { algorithms, defaultChecksumData, @@ -214,6 +215,12 @@ function createAndStoreObject( // Always set originOp metadataStoreParams.originOp = originOp; + // Data stays in hot storage: the transition is triggered async from the oplog event + if (!isDeleteMarker && !isPutVersion && isColdStorageClass(request.headers['x-amz-storage-class'])) { + metadataStoreParams.amzStorageClass = request.headers['x-amz-storage-class']; + metadataStoreParams.transitionInProgress = true; + } + if (!isDeleteMarker) { metadataStoreParams.contentType = request.headers['content-type']; metadataStoreParams.cacheControl = request.headers['cache-control']; diff --git a/lib/api/apiUtils/object/storageClass.js b/lib/api/apiUtils/object/storageClass.js new file mode 100644 index 0000000000..3de288437f --- /dev/null +++ b/lib/api/apiUtils/object/storageClass.js @@ -0,0 +1,50 @@ +const { errors } = require('arsenal'); + +const constants = require('../../../../constants'); +const { config } = require('../../../Config'); + +/** + * Whether the given storage class names a cold location of this deployment + * @param {string} storageClass - value of the x-amz-storage-class header + * @returns {boolean|undefined} true if the storage class is a cold location + */ +function isColdStorageClass(storageClass) { + return config.locationConstraints[storageClass]?.isCold; +} + +/** + * Validate the x-amz-storage-class header of an object write request. + * + * Accepted values are the regular S3 storage classes, and, with + * `enableDirectToCold`, the name of a cold location of this deployment. + * + * Restricting which identity may use a given storage class is done through the + * `s3:x-amz-storage-class` IAM policy condition key, and so is not handled here. + * + * @param {object} headers - request headers + * @returns {ArsenalError|null} InvalidStorageClass if the value is not supported + */ +function validateStorageClass(headers) { + const storageClass = headers['x-amz-storage-class']; + if (!storageClass) { + return null; + } + if (constants.validStorageClasses.includes(storageClass)) { + return null; + } + // A restore writes back an object which is already in a cold location: naming a cold + // storage class would send it straight back, so it is rejected rather than ignored. + const putVersionId = headers['x-scal-s3-version-id']; + if (putVersionId || putVersionId === '') { + return errors.InvalidStorageClass; + } + if (config.enableDirectToCold && isColdStorageClass(storageClass)) { + return null; + } + return errors.InvalidStorageClass; +} + +module.exports = { + isColdStorageClass, + validateStorageClass, +}; diff --git a/lib/api/completeMultipartUpload.js b/lib/api/completeMultipartUpload.js index 766e4bc844..9ff04bd29e 100644 --- a/lib/api/completeMultipartUpload.js +++ b/lib/api/completeMultipartUpload.js @@ -24,6 +24,7 @@ const { validateAndFilterMpuParts, generateMpuPartStorageInfo } = s3middleware.p const locationKeysHaveChanged = require('./apiUtils/object/locationKeysHaveChanged'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); +const { isColdStorageClass } = require('./apiUtils/object/storageClass'); const { validateQuotas } = require('./apiUtils/quotas/quotaUtils'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const { @@ -816,6 +817,12 @@ function completeMultipartUpload(authInfo, request, log, callback) { overheadField: constants.overheadField, log, }; + // Data stays in hot storage: the transition is triggered async from the oplog event + if (!isPutVersion && isColdStorageClass(storedMetadata['x-amz-storage-class'])) { + metaStoreParams.amzStorageClass = storedMetadata['x-amz-storage-class']; + metaStoreParams.transitionInProgress = true; + } + // Persist FULL_OBJECT final-object checksum on the new ObjectMD. // COMPOSITE is intentionally skipped to prevent metadata bloat, // to be done in S3C-10399. diff --git a/lib/api/initiateMultipartUpload.js b/lib/api/initiateMultipartUpload.js index a216290096..e3637181e8 100644 --- a/lib/api/initiateMultipartUpload.js +++ b/lib/api/initiateMultipartUpload.js @@ -16,6 +16,7 @@ const validateWebsiteHeader = require('./apiUtils/object/websiteServing').valida const monitoring = require('../utilities/monitoringHandler'); const { data } = require('../data/wrapper'); const applyZenkoUserMD = require('./apiUtils/object/applyZenkoUserMD'); +const { validateStorageClass } = require('./apiUtils/object/storageClass'); const { validateHeaders, compareObjectLockInformation } = require('./apiUtils/object/objectLockHelpers'); const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); @@ -75,13 +76,11 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // name itself. To prevent this, we are restricting the creation of a // multipart upload object with a key containing the splitter. const websiteRedirectHeader = request.headers['x-amz-website-redirect-location']; - if ( - request.headers['x-amz-storage-class'] && - !constants.validStorageClasses.includes(request.headers['x-amz-storage-class']) - ) { + const storageClassError = validateStorageClass(request.headers); + if (storageClassError) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', bucketName, errorInstances.InvalidStorageClass.code, 'initiateMultipartUpload'); - return callback(errors.InvalidStorageClass); + monitoring.promMetrics('PUT', bucketName, storageClassError.code, 'initiateMultipartUpload'); + return callback(storageClassError); } if (!validateWebsiteHeader(websiteRedirectHeader)) { const err = errors.InvalidRedirectLocation; @@ -107,15 +106,7 @@ function initiateMultipartUpload(authInfo, request, log, callback) { // field on the object applyZenkoUserMD(metaHeaders); - // TODO: Add this as a utility function for all object put requests - // but after authentication so that string to sign is not impacted - // This is GH Issue#89 - // TODO: remove in CLDSRV-639 - const storageClassOptions = ['standard', 'standard_ia', 'reduced_redundancy']; - let storageClass = 'STANDARD'; - if (storageClassOptions.indexOf(request.headers['x-amz-storage-class']) > -1) { - storageClass = request.headers['x-amz-storage-class'].toUpperCase(); - } + const storageClass = request.headers['x-amz-storage-class'] || 'STANDARD'; const metadataValParams = { objectKey, authInfo, diff --git a/lib/api/objectCopy.js b/lib/api/objectCopy.js index 153aa21f03..128862950c 100644 --- a/lib/api/objectCopy.js +++ b/lib/api/objectCopy.js @@ -19,6 +19,7 @@ const validateWebsiteHeader = require('./apiUtils/object/websiteServing').valida const { config } = require('../Config'); const monitoring = require('../utilities/monitoringHandler'); const applyZenkoUserMD = require('./apiUtils/object/applyZenkoUserMD'); +const { isColdStorageClass, validateStorageClass } = require('./apiUtils/object/storageClass'); const { getObjectSSEConfiguration } = require('./apiUtils/bucket/bucketEncryption'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { verifyColdObjectAvailable } = require('./apiUtils/object/coldStorage'); @@ -295,7 +296,6 @@ function _prepMetadata( overrideMetadata['x-amz-server-side-encryption'] = headers['x-amz-server-side-encryption']; } if (headers['x-amz-storage-class']) { - // TODO: remove in CLDSRV-639 overrideMetadata['x-amz-storage-class'] = headers['x-amz-storage-class']; } if (headers['x-amz-website-redirect-location']) { @@ -409,6 +409,12 @@ function _prepMetadata( originOp: 's3:ObjectCreated:Copy', }; + // Data stays in hot storage: the transition is triggered async from the oplog event. + // The storage class itself is already carried over through overrideMetadata. + if (isColdStorageClass(headers['x-amz-storage-class'])) { + storeMetadataParams.transitionInProgress = true; + } + const defaultRetentionConfig = destBucketMD.getObjectLockConfiguration(); if (defaultRetentionConfig && !legalHoldHeader) { storeMetadataParams.defaultRetention = defaultRetentionConfig; @@ -493,13 +499,11 @@ function objectCopy(authInfo, request, sourceBucket, sourceObject, sourceVersion const websiteRedirectHeader = request.headers['x-amz-website-redirect-location']; const responseHeaders = {}; - if ( - request.headers['x-amz-storage-class'] && - !constants.validStorageClasses.includes(request.headers['x-amz-storage-class']) - ) { + const storageClassError = validateStorageClass(request.headers); + if (storageClassError) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', destBucketName, errorInstances.InvalidStorageClass.code, 'copyObject'); - return callback(errors.InvalidStorageClass); + monitoring.promMetrics('PUT', destBucketName, storageClassError.code, 'copyObject'); + return callback(storageClassError); } if (!validateWebsiteHeader(websiteRedirectHeader)) { const err = errors.InvalidRedirectLocation; diff --git a/lib/api/objectPut.js b/lib/api/objectPut.js index d226729ffd..fa8a54667f 100644 --- a/lib/api/objectPut.js +++ b/lib/api/objectPut.js @@ -16,6 +16,7 @@ const { hasNonPrintables } = require('../utilities/stringChecks'); const kms = require('../kms/wrapper'); const monitoring = require('../utilities/monitoringHandler'); const { validatePutVersionId } = require('./apiUtils/object/coldStorage'); +const { validateStorageClass } = require('./apiUtils/object/storageClass'); const { setExpirationHeaders } = require('./apiUtils/object/expirationHeaders'); const { setSSEHeaders } = require('./apiUtils/object/sseHeaders'); const validatePayloadProtocol = require('./apiUtils/object/validatePayloadProtocol'); @@ -71,10 +72,11 @@ function objectPut(authInfo, request, streamingV4Params, log, callback) { } const { bucketName, headers, method, objectKey, parsedContentLength, query } = request; - if (headers['x-amz-storage-class'] && !constants.validStorageClasses.includes(headers['x-amz-storage-class'])) { + const storageClassError = validateStorageClass(headers); + if (storageClassError) { log.trace('invalid storage-class header'); - monitoring.promMetrics('PUT', request.bucketName, errorInstances.InvalidStorageClass.code, 'putObject'); - return callback(errors.InvalidStorageClass); + monitoring.promMetrics('PUT', request.bucketName, storageClassError.code, 'putObject'); + return callback(storageClassError); } if (!aclUtils.checkGrantHeaderValidity(headers)) { log.trace('invalid acl header'); diff --git a/lib/services.js b/lib/services.js index 2eefec878a..09895964fb 100644 --- a/lib/services.js +++ b/lib/services.js @@ -138,6 +138,7 @@ const services = { oldReplayId, deleteNullKey, amzStorageClass, + transitionInProgress, overheadField, needOplogUpdate, restoredEtag, @@ -223,9 +224,14 @@ const services = { if (updateMicroVersionId) { md.updateMicroVersionId(config.instanceId, config.replicationGroupId); } + if (amzStorageClass) { + md.setAmzStorageClass(amzStorageClass); + } + if (transitionInProgress) { + md.setTransitionInProgress(true, Date.now()); + } // update restore if (archive) { - md.setAmzStorageClass(amzStorageClass); md.setArchive( new ObjectMDArchive( archive.archiveInfo, @@ -581,7 +587,7 @@ const services = { multipartObjectMD['content-encoding'] = removeAWSChunked(params.headers['content-encoding']); multipartObjectMD['content-type'] = params.headers['content-type']; multipartObjectMD.expires = params.headers.expires; - multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639 + multipartObjectMD['x-amz-storage-class'] = params.storageClass; multipartObjectMD['x-amz-website-redirect-location'] = params.headers['x-amz-website-redirect-location']; if (cipherBundle) { multipartObjectMD['x-amz-server-side-encryption'] = cipherBundle.algorithm; diff --git a/tests/unit/api/apiUtils/storageClass.js b/tests/unit/api/apiUtils/storageClass.js new file mode 100644 index 0000000000..04c5b12be3 --- /dev/null +++ b/tests/unit/api/apiUtils/storageClass.js @@ -0,0 +1,98 @@ +const assert = require('assert'); + +const { isColdStorageClass, validateStorageClass } = require('../../../../lib/api/apiUtils/object/storageClass'); +const { config } = require('../../../../lib/Config'); + +const coldLocation = 'location-dmf-v1'; +const hotLocation = 'us-east-1'; + +describe('storage class helpers', () => { + let originalEnableDirectToCold; + + beforeEach(() => { + originalEnableDirectToCold = config.enableDirectToCold; + }); + + afterEach(() => { + config.enableDirectToCold = originalEnableDirectToCold; + }); + + describe('isColdStorageClass', () => { + it('should return true for a cold location', () => { + assert.strictEqual(isColdStorageClass(coldLocation), true); + }); + + it('should not return true for a hot location', () => { + assert.ok(!isColdStorageClass(hotLocation)); + }); + + it('should not return true for a regular storage class', () => { + assert.ok(!isColdStorageClass('STANDARD')); + }); + + it('should not return true for an unknown location', () => { + assert.ok(!isColdStorageClass('does-not-exist')); + }); + + it('should not return true when no storage class is given', () => { + assert.ok(!isColdStorageClass(undefined)); + }); + }); + + describe('validateStorageClass', () => { + it('should accept a request without a storage class header', () => { + assert.strictEqual(validateStorageClass({}), null); + }); + + it('should accept a regular storage class', () => { + assert.strictEqual(validateStorageClass({ 'x-amz-storage-class': 'STANDARD' }), null); + }); + + it('should reject a cold location when the option is disabled', () => { + config.enableDirectToCold = false; + const err = validateStorageClass({ 'x-amz-storage-class': coldLocation }); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should accept a cold location when the option is enabled', () => { + config.enableDirectToCold = true; + assert.strictEqual(validateStorageClass({ 'x-amz-storage-class': coldLocation }), null); + }); + + it('should reject a hot location even when the option is enabled', () => { + config.enableDirectToCold = true; + const err = validateStorageClass({ 'x-amz-storage-class': hotLocation }); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should reject an unknown storage class', () => { + config.enableDirectToCold = true; + const err = validateStorageClass({ 'x-amz-storage-class': 'GLACIER' }); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should reject a cold location on a restore', () => { + config.enableDirectToCold = true; + const err = validateStorageClass({ + 'x-amz-storage-class': coldLocation, + 'x-scal-s3-version-id': 'some-version-id', + }); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should reject a cold location on a restore of a non-versioned object', () => { + config.enableDirectToCold = true; + const err = validateStorageClass({ + 'x-amz-storage-class': coldLocation, + 'x-scal-s3-version-id': '', + }); + assert.strictEqual(err.message, 'InvalidStorageClass'); + }); + + it('should accept a regular storage class on a restore', () => { + config.enableDirectToCold = true; + const headers = { 'x-amz-storage-class': 'STANDARD', 'x-scal-s3-version-id': '' }; + assert.strictEqual(validateStorageClass(headers), null); + }); + }); +}); diff --git a/tests/unit/api/directToCold.js b/tests/unit/api/directToCold.js new file mode 100644 index 0000000000..4d14e33072 --- /dev/null +++ b/tests/unit/api/directToCold.js @@ -0,0 +1,363 @@ +const assert = require('assert'); +const async = require('async'); + +const { bucketPut } = require('../../../lib/api/bucketPut'); +const objectPut = require('../../../lib/api/objectPut'); +const objectCopy = require('../../../lib/api/objectCopy'); +const initiateMultipartUpload = require('../../../lib/api/initiateMultipartUpload'); +const DummyRequest = require('../DummyRequest'); +const { cleanup, DummyRequestLogger, makeAuthInfo } = require('../helpers'); +const metadata = require('../metadataswitch'); +const mpuUtils = require('../utils/mpuUtils'); +const { config } = require('../../../lib/Config'); + +const log = new DummyRequestLogger(); +const authInfo = makeAuthInfo('accessKey1'); +const namespace = 'default'; +const bucketName = 'bucketname'; +const objectKey = 'objectName'; +const postBody = Buffer.from('I am a body', 'utf8'); +const coldLocation = 'location-dmf-v1'; +const hotLocation = 'scality-internal-mem'; +// marks a request as a restore, writing back an object already stored in a cold location +const putVersionHeader = { 'x-scal-s3-version-id': '' }; + +const putBucketRequest = new DummyRequest({ + bucketName, + namespace, + headers: { host: `${bucketName}.s3.amazonaws.com` }, + url: '/', + post: + '' + + `${hotLocation}` + + '', +}); + +function putObjectRequest(headers = {}) { + return new DummyRequest( + { + bucketName, + namespace, + objectKey, + headers: { host: `${bucketName}.s3.amazonaws.com`, ...headers }, + url: `/${bucketName}/${objectKey}`, + }, + postBody, + ); +} + +function copyObjectRequest(headers = {}) { + return new DummyRequest({ + bucketName, + namespace, + objectKey: 'copiedObject', + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-copy-source': `/${bucketName}/${objectKey}`, + ...headers, + }, + url: `/${bucketName}/copiedObject`, + socket: {}, + }); +} + +function getObjectMD(key, cb) { + return metadata.getObjectMD(bucketName, key, {}, log, cb); +} + +function assertDirectToCold(md) { + assert.strictEqual(md['x-amz-storage-class'], coldLocation); + // the data itself stays in the hot location + assert.strictEqual(md.dataStoreName, hotLocation); + // the transition has not happened yet, so there is no archive info + assert.strictEqual(md.archive, undefined); + assert.strictEqual(md['x-amz-scal-transition-in-progress'], true); + assert.match(md['x-amz-scal-transition-time'], /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/); +} + +function assertNotTransitioned(md) { + assert.strictEqual(md.dataStoreName, hotLocation); + assert.strictEqual(md.archive, undefined); + assert.strictEqual(md['x-amz-scal-transition-in-progress'], undefined); +} + +describe('direct to cold', () => { + let originalEnableDirectToCold; + + beforeEach(done => { + originalEnableDirectToCold = config.enableDirectToCold; + cleanup(); + bucketPut(authInfo, putBucketRequest, log, done); + }); + + afterEach(() => { + config.enableDirectToCold = originalEnableDirectToCold; + cleanup(); + }); + + describe('when disabled', () => { + beforeEach(() => { + config.enableDirectToCold = false; + }); + + it('should reject a PUT naming a cold location', done => { + objectPut(authInfo, putObjectRequest({ 'x-amz-storage-class': coldLocation }), undefined, log, err => { + assert.strictEqual(err.message, 'InvalidStorageClass'); + done(); + }); + }); + + it('should reject a CreateMultipartUpload naming a cold location', done => { + const request = mpuUtils.createinitiateMPURequest(namespace, bucketName, objectKey, { + 'x-amz-storage-class': coldLocation, + }); + initiateMultipartUpload(authInfo, request, log, err => { + assert.strictEqual(err.message, 'InvalidStorageClass'); + done(); + }); + }); + + it('should reject a CopyObject naming a cold location', done => { + async.series( + [ + next => objectPut(authInfo, putObjectRequest(), undefined, log, next), + next => + objectCopy( + authInfo, + copyObjectRequest({ 'x-amz-storage-class': coldLocation }), + bucketName, + objectKey, + undefined, + log, + next, + ), + ], + err => { + assert.strictEqual(err.message, 'InvalidStorageClass'); + done(); + }, + ); + }); + }); + + describe('when enabled', () => { + beforeEach(() => { + config.enableDirectToCold = true; + }); + + it('should flag an object PUT with a cold storage class for transition', done => { + async.waterfall( + [ + next => + objectPut( + authInfo, + putObjectRequest({ 'x-amz-storage-class': coldLocation }), + undefined, + log, + err => next(err), + ), + next => getObjectMD(objectKey, next), + ], + (err, md) => { + assert.ifError(err); + assertDirectToCold(md); + assert.strictEqual(md.originOp, 's3:ObjectCreated:Put'); + done(); + }, + ); + }); + + it('should not flag an object PUT without a storage class', done => { + async.waterfall( + [ + next => objectPut(authInfo, putObjectRequest(), undefined, log, err => next(err)), + next => getObjectMD(objectKey, next), + ], + (err, md) => { + assert.ifError(err); + assertNotTransitioned(md); + assert.strictEqual(md['x-amz-storage-class'], 'STANDARD'); + done(); + }, + ); + }); + + it('should flag a completed multipart upload with a cold storage class for transition', done => { + async.waterfall( + [ + next => + mpuUtils + .initiateMpuP(bucketName, namespace, objectKey, log, { + 'x-amz-storage-class': coldLocation, + }) + .then(uploadId => next(null, uploadId), next), + (uploadId, next) => + mpuUtils + .uploadPartP(bucketName, namespace, objectKey, uploadId, log) + .then(() => next(null, uploadId), next), + (uploadId, next) => + mpuUtils.completeMpuP(bucketName, namespace, objectKey, uploadId, log).then(() => next(), next), + next => getObjectMD(objectKey, next), + ], + (err, md) => { + assert.ifError(err); + assertDirectToCold(md); + assert.strictEqual(md.originOp, 's3:ObjectCreated:CompleteMultipartUpload'); + done(); + }, + ); + }); + + it('should flag a copied object with a cold storage class for transition', done => { + async.waterfall( + [ + next => objectPut(authInfo, putObjectRequest(), undefined, log, err => next(err)), + next => + objectCopy( + authInfo, + copyObjectRequest({ 'x-amz-storage-class': coldLocation }), + bucketName, + objectKey, + undefined, + log, + err => next(err), + ), + next => getObjectMD('copiedObject', next), + ], + (err, md) => { + assert.ifError(err); + assertDirectToCold(md); + assert.strictEqual(md.originOp, 's3:ObjectCreated:Copy'); + done(); + }, + ); + }); + + it('should flag a self-copy changing only the storage class, and keep the data in place', done => { + const selfCopyRequest = new DummyRequest({ + bucketName, + namespace, + objectKey, + headers: { + host: `${bucketName}.s3.amazonaws.com`, + 'x-amz-copy-source': `/${bucketName}/${objectKey}`, + 'x-amz-storage-class': coldLocation, + }, + url: `/${bucketName}/${objectKey}`, + socket: {}, + }); + async.waterfall( + [ + next => objectPut(authInfo, putObjectRequest(), undefined, log, err => next(err)), + next => getObjectMD(objectKey, next), + (sourceMD, next) => + objectCopy(authInfo, selfCopyRequest, bucketName, objectKey, undefined, log, err => + next(err, sourceMD), + ), + (sourceMD, next) => getObjectMD(objectKey, (err, md) => next(err, sourceMD, md)), + ], + (err, sourceMD, md) => { + assert.ifError(err); + assertDirectToCold(md); + // the bytes are not rewritten: the existing data locations are reused + assert.deepStrictEqual( + md.location.map(l => l.key), + sourceMD.location.map(l => l.key), + ); + done(); + }, + ); + }); + + it('should reject a restore naming a cold location', done => { + const request = putObjectRequest({ + 'x-amz-storage-class': coldLocation, + ...putVersionHeader, + }); + objectPut(authInfo, request, undefined, log, err => { + assert.strictEqual(err.message, 'InvalidStorageClass'); + done(); + }); + }); + + it('should reject a restore initiating a multipart upload naming a cold location', done => { + const request = mpuUtils.createinitiateMPURequest(namespace, bucketName, objectKey, { + 'x-amz-storage-class': coldLocation, + ...putVersionHeader, + }); + initiateMultipartUpload(authInfo, request, log, err => { + assert.strictEqual(err.message, 'InvalidStorageClass'); + done(); + }); + }); + + it('should not flag a restore completing a multipart upload', done => { + async.waterfall( + [ + next => objectPut(authInfo, putObjectRequest(), undefined, log, err => next(err)), + next => getObjectMD(objectKey, next), + // simulate an object whose restore from the cold location is in progress + (md, next) => { + /* eslint-disable no-param-reassign */ + md['x-amz-storage-class'] = coldLocation; + md.dataStoreName = coldLocation; + md.archive = { + archiveInfo: { archiveId: 'archive-id' }, + restoreRequestedAt: new Date().toString(), + restoreRequestedDays: 5, + }; + /* eslint-enable no-param-reassign */ + metadata.putObjectMD(bucketName, objectKey, md, {}, log, err => next(err)); + }, + next => + mpuUtils + .initiateMpuP(bucketName, namespace, objectKey, log, { + 'x-amz-storage-class': coldLocation, + }) + .then(uploadId => next(null, uploadId), next), + (uploadId, next) => + mpuUtils + .uploadPartP(bucketName, namespace, objectKey, uploadId, log) + .then(() => next(null, uploadId), next), + (uploadId, next) => + mpuUtils + .completeMpuP(bucketName, namespace, objectKey, uploadId, log, { + extraHeaders: putVersionHeader, + }) + .then(() => next(), next), + next => getObjectMD(objectKey, next), + ], + (err, md) => { + assert.ifError(err); + // the object is being restored, it must not be transitioned back to cold + assert.strictEqual(md['x-amz-scal-transition-in-progress'], undefined); + done(); + }, + ); + }); + + it('should not flag the source object of a copy', done => { + async.waterfall( + [ + next => objectPut(authInfo, putObjectRequest(), undefined, log, err => next(err)), + next => + objectCopy( + authInfo, + copyObjectRequest({ 'x-amz-storage-class': coldLocation }), + bucketName, + objectKey, + undefined, + log, + err => next(err), + ), + next => getObjectMD(objectKey, next), + ], + (err, md) => { + assert.ifError(err); + assertNotTransitioned(md); + done(); + }, + ); + }); + }); +});