From a76854e6c50cbedd937e2e246423c0707514dcb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Wed, 3 Jun 2026 16:54:08 +0200 Subject: [PATCH 1/5] Reformat with prettier Issue: CLDSRV-906 --- lib/api/objectDeleteTagging.js | 145 ++-- lib/api/objectPutLegalHold.js | 152 ++-- lib/api/objectPutRetention.js | 183 ++-- lib/api/objectPutTagging.js | 154 ++-- lib/metadata/acl.js | 88 +- lib/services.js | 615 +++++++------- lib/utilities/collectResponseHeaders.js | 66 +- tests/unit/api/objectReplicationMD.js | 926 +++++++++++---------- tests/unit/utils/collectResponseHeaders.js | 23 +- 9 files changed, 1204 insertions(+), 1148 deletions(-) diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 71115ffe5a..45d67a0f97 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -1,8 +1,11 @@ const async = require('async'); const { errors } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } - = require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -48,75 +51,81 @@ function objectDeleteTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectDeleteTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectDeleteTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectDeleteTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.tags = {}; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectDeleteTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectDeleteTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectDeleteTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + objectMD.tags = {}; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + // eslint-disable-next-line no-param-reassign + objectMD.originOp = 's3:ObjectTagging:Delete'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + (bucket, objectMD, next) => + // if external backends handles tagging + data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, log, err => + next(err, bucket, objectMD), + ), + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectDeleteTagging' }); + monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteObjectTagging'); + } else { + pushMetric('deleteObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectTagging:Delete'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => - next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - (bucket, objectMD, next) => - // if external backends handles tagging - data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, - log, err => next(err, bucket, objectMD)), - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectDeleteTagging' }); - monitoring.promMetrics( - 'DELETE', bucketName, err.code, 'deleteObjectTagging'); - } else { - pushMetric('deleteObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'DELETE', bucketName, '200', 'deleteObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectDeleteTagging; diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index c16f2c84e8..d687f77ce6 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -2,8 +2,11 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -47,78 +50,87 @@ function objectPutLegalHold(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutLegalHold', error: err }); + return next(err); + } + if (!objectMD) { + const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutLegalHold', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutLegalHold' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectPutLegalHold' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + bucket, + ); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + log.trace('parsing legal hold'); + parseLegalHoldXml(request.post, log, (err, res) => next(err, bucket, res, objectMD)); + }, + (bucket, legalHold, objectMD, next) => { + // eslint-disable-next-line no-param-reassign + objectMD.legalHold = legalHold; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + // eslint-disable-next-line no-param-reassign + objectMD.originOp = 's3:ObjectLegalHold:Put'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); if (err) { - log.trace('request authorization failed', - { method: 'objectPutLegalHold', error: err }); - return next(err); - } - if (!objectMD) { - const err = versionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutLegalHold', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutLegalHold' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectPutLegalHold' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration' - ), bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - log.trace('parsing legal hold'); - parseLegalHoldXml(request.post, log, (err, res) => - next(err, bucket, res, objectMD)); - }, - (bucket, legalHold, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.legalHold = legalHold; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { - // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + log.trace('error processing request', { error: err, method: 'objectPutLegalHold' }); + } else { + pushMetric('putObjectLegalHold', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectLegalHold:Put'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'objectPutLegalHold' }); - } else { - pushMetric('putObjectLegalHold', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutLegalHold; diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index 6a7a2c8441..b8182646e9 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -1,10 +1,12 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); -const { ObjectLockInfo, hasGovernanceBypassHeader } = - require('./apiUtils/object/objectLockHelpers'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); +const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object/objectLockHelpers'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); @@ -50,99 +52,106 @@ function objectPutRetention(authInfo, request, log, callback) { const hasGovernanceBypass = hasGovernanceBypassHeader(request.headers); - return async.waterfall([ - next => { - log.trace('parsing retention information'); - parseRetentionXml(request.post, log, - (err, retentionInfo) => { + return async.waterfall( + [ + next => { + log.trace('parsing retention information'); + parseRetentionXml(request.post, log, (err, retentionInfo) => { if (err) { - log.trace('error parsing retention information', - { error: err }); + log.trace('error parsing retention information', { error: err }); return next(err); } - const remainingDays = Math.ceil( - (new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24)); + const remainingDays = Math.ceil((new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24)); metadataValParams.request.objectLockRetentionDays = remainingDays; return next(null, retentionInfo); }); - }, - (retentionInfo, next) => standardMetadataValidateBucketAndObj(metadataValParams, - request.actionImplicitDenies, log, (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectPutRetention', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutRetention', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutRetention' }); - return next(errors.MethodNotAllowed, bucket); - } - if (!bucket.isObjectLockEnabled()) { - log.trace('object lock not enabled on bucket', - { method: 'objectPutRetention' }); - return next(errorInstances.InvalidRequest.customizeDescription( - 'Bucket is missing Object Lock Configuration' - ), bucket); - } - return next(null, bucket, retentionInfo, objectMD); - }), - (bucket, retentionInfo, objectMD, next) => { - const objLockInfo = new ObjectLockInfo({ - mode: objectMD.retentionMode, - date: objectMD.retentionDate, - legalHold: objectMD.legalHold, - }); + }, + (retentionInfo, next) => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutRetention', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutRetention', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutRetention' }); + return next(errors.MethodNotAllowed, bucket); + } + if (!bucket.isObjectLockEnabled()) { + log.trace('object lock not enabled on bucket', { method: 'objectPutRetention' }); + return next( + errorInstances.InvalidRequest.customizeDescription( + 'Bucket is missing Object Lock Configuration', + ), + bucket, + ); + } + return next(null, bucket, retentionInfo, objectMD); + }, + ), + (bucket, retentionInfo, objectMD, next) => { + const objLockInfo = new ObjectLockInfo({ + mode: objectMD.retentionMode, + date: objectMD.retentionDate, + legalHold: objectMD.legalHold, + }); - if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) { - return next(errors.AccessDenied, bucket); - } + if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) { + return next(errors.AccessDenied, bucket); + } - return next(null, bucket, retentionInfo, objectMD); - }, - (bucket, retentionInfo, objectMD, next) => { - /* eslint-disable no-param-reassign */ - objectMD.retentionMode = retentionInfo.mode; - objectMD.retentionDate = retentionInfo.date; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + return next(null, bucket, retentionInfo, objectMD); + }, + (bucket, retentionInfo, objectMD, next) => { + /* eslint-disable no-param-reassign */ + objectMD.retentionMode = retentionInfo.mode; + objectMD.retentionDate = retentionInfo.date; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } + objectMD.originOp = 's3:ObjectRetention:Put'; + /* eslint-enable no-param-reassign */ + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectPutRetention' }); + } else { + pushMetric('putObjectRetention', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - objectMD.originOp = 's3:ObjectRetention:Put'; - /* eslint-enable no-param-reassign */ - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', - { error: err, method: 'objectPutRetention' }); - } else { - pushMetric('putObjectRetention', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutRetention; diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index ef23dcf64d..85bb652787 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -1,8 +1,11 @@ const async = require('async'); const { errors, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = - require('./apiUtils/object/versioning'); +const { + decodeVersionId, + getVersionIdResHeader, + getVersionSpecificMetadataOptions, +} = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -47,80 +50,85 @@ function objectPutTagging(authInfo, request, log, callback) { request, }; - return async.waterfall([ - next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log, - (err, bucket, objectMD) => { - if (err) { - log.trace('request authorization failed', - { method: 'objectPutTagging', error: err }); - return next(err); - } - if (!objectMD) { - const err = reqVersionId ? errors.NoSuchVersion : - errors.NoSuchKey; - log.trace('error no object metadata found', - { method: 'objectPutTagging', error: err }); - return next(err, bucket); - } - if (objectMD.isDeleteMarker) { - log.trace('version is a delete marker', - { method: 'objectPutTagging' }); - // FIXME we should return a `x-amz-delete-marker: true` header, - // see S3C-7592 - return next(errors.MethodNotAllowed, bucket); - } - return next(null, bucket, objectMD); - }), - (bucket, objectMD, next) => { - log.trace('parsing tag(s)'); - parseTagXml(request.post, log, (err, tags) => - next(err, bucket, tags, objectMD)); - }, - (bucket, tags, objectMD, next) => { - // eslint-disable-next-line no-param-reassign - objectMD.tags = tags; - const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); - const replicationInfo = getReplicationInfo(config, - objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD); - if (replicationInfo) { + return async.waterfall( + [ + next => + standardMetadataValidateBucketAndObj( + metadataValParams, + request.actionImplicitDenies, + log, + (err, bucket, objectMD) => { + if (err) { + log.trace('request authorization failed', { method: 'objectPutTagging', error: err }); + return next(err); + } + if (!objectMD) { + const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey; + log.trace('error no object metadata found', { method: 'objectPutTagging', error: err }); + return next(err, bucket); + } + if (objectMD.isDeleteMarker) { + log.trace('version is a delete marker', { method: 'objectPutTagging' }); + // FIXME we should return a `x-amz-delete-marker: true` header, + // see S3C-7592 + return next(errors.MethodNotAllowed, bucket); + } + return next(null, bucket, objectMD); + }, + ), + (bucket, objectMD, next) => { + log.trace('parsing tag(s)'); + parseTagXml(request.post, log, (err, tags) => next(err, bucket, tags, objectMD)); + }, + (bucket, tags, objectMD, next) => { + // eslint-disable-next-line no-param-reassign + objectMD.tags = tags; + const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + const replicationInfo = getReplicationInfo( + config, + objectKey, + bucket, + true, + 0, + REPLICATION_ACTION, + objectMD, + ); + if (replicationInfo) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); + } // eslint-disable-next-line no-param-reassign - objectMD.replicationInfo = Object.assign({}, - objectMD.replicationInfo, replicationInfo); + objectMD.originOp = 's3:ObjectTagging:Put'; + metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => + next(err, bucket, objectMD), + ); + }, + (bucket, objectMD, next) => + // if external backend handles tagging + data.objectTagging('Put', objectKey, bucket.getName(), objectMD, log, err => + next(err, bucket, objectMD), + ), + ], + (err, bucket, objectMD) => { + const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket); + if (err) { + log.trace('error processing request', { error: err, method: 'objectPutTagging' }); + monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectTagging'); + } else { + pushMetric('putObjectTagging', log, { + authInfo, + bucket: bucketName, + keys: [objectKey], + versionId: objectMD ? objectMD.versionId : undefined, + location: objectMD ? objectMD.dataStoreName : undefined, + }); + monitoring.promMetrics('PUT', bucketName, '200', 'putObjectTagging'); + const verCfg = bucket.getVersioningConfiguration(); + additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD); } - // eslint-disable-next-line no-param-reassign - objectMD.originOp = 's3:ObjectTagging:Put'; - metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, - log, err => - next(err, bucket, objectMD)); + return callback(err, additionalResHeaders); }, - (bucket, objectMD, next) => - // if external backend handles tagging - data.objectTagging('Put', objectKey, bucket.getName(), objectMD, - log, err => next(err, bucket, objectMD)), - ], (err, bucket, objectMD) => { - const additionalResHeaders = collectCorsHeaders(request.headers.origin, - request.method, bucket); - if (err) { - log.trace('error processing request', { error: err, - method: 'objectPutTagging' }); - monitoring.promMetrics('PUT', bucketName, err.code, - 'putObjectTagging'); - } else { - pushMetric('putObjectTagging', log, { - authInfo, - bucket: bucketName, - keys: [objectKey], - versionId: objectMD ? objectMD.versionId : undefined, - location: objectMD ? objectMD.dataStoreName : undefined, - }); - monitoring.promMetrics( - 'PUT', bucketName, '200', 'putObjectTagging'); - const verCfg = bucket.getVersioningConfiguration(); - additionalResHeaders['x-amz-version-id'] = - getVersionIdResHeader(verCfg, objectMD); - } - return callback(err, additionalResHeaders); - }); + ); } module.exports = objectPutTagging; diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index f48ab7aa42..961b95d79f 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -31,14 +31,16 @@ const acl = { * contain the same number of elements, and all elements from one * grant are incuded in the other grant */ - return oldAcl[grant].length === newAcl[grant].length - && oldAcl[grant].every(value => newAcl[grant].includes(value)); + return ( + oldAcl[grant].length === newAcl[grant].length && oldAcl[grant].every(value => newAcl[grant].includes(value)) + ); }, addObjectACL(bucket, objectKey, objectMD, addACLParams, params, log, cb) { log.trace('updating object acl in metadata'); - const isAclUnchanged = Object.keys(objectMD.acl).length === Object.keys(addACLParams).length - && Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams)); + const isAclUnchanged = + Object.keys(objectMD.acl).length === Object.keys(addACLParams).length && + Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams)); if (!isAclUnchanged) { /* eslint-disable no-param-reassign */ objectMD.acl = addACLParams; @@ -77,14 +79,22 @@ const acl = { }; let validCannedACL = []; if (resourceType === 'bucket') { - validCannedACL = - ['private', 'public-read', 'public-read-write', - 'authenticated-read', 'log-delivery-write']; + validCannedACL = [ + 'private', + 'public-read', + 'public-read-write', + 'authenticated-read', + 'log-delivery-write', + ]; } else if (resourceType === 'object') { - validCannedACL = - ['private', 'public-read', 'public-read-write', - 'authenticated-read', 'bucket-owner-read', - 'bucket-owner-full-control']; + validCannedACL = [ + 'private', + 'public-read', + 'public-read-write', + 'authenticated-read', + 'bucket-owner-read', + 'bucket-owner-full-control', + ]; } // parse canned acl @@ -98,45 +108,34 @@ const acl = { } // parse grant headers - const grantReadHeader = - aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ'); + const grantReadHeader = aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ'); let grantWriteHeader = []; if (resourceType === 'bucket') { - grantWriteHeader = aclUtils - .parseGrant(headers['x-amz-grant-write'], 'WRITE'); + grantWriteHeader = aclUtils.parseGrant(headers['x-amz-grant-write'], 'WRITE'); } - const grantReadACPHeader = aclUtils - .parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP'); - const grantWriteACPHeader = aclUtils - .parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP'); - const grantFullControlHeader = aclUtils - .parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL'); - const allGrantHeaders = - [].concat(grantReadHeader, grantWriteHeader, - grantReadACPHeader, grantWriteACPHeader, - grantFullControlHeader).filter(item => item !== undefined); + const grantReadACPHeader = aclUtils.parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP'); + const grantWriteACPHeader = aclUtils.parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP'); + const grantFullControlHeader = aclUtils.parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL'); + const allGrantHeaders = [] + .concat(grantReadHeader, grantWriteHeader, grantReadACPHeader, grantWriteACPHeader, grantFullControlHeader) + .filter(item => item !== undefined); if (allGrantHeaders.length === 0) { return cb(null, currentResourceACL); } - const usersIdentifiedByEmail = allGrantHeaders - .filter(it => it && it.userIDType.toLowerCase() === 'emailaddress'); - const usersIdentifiedByGroup = allGrantHeaders - .filter(item => item && item.userIDType.toLowerCase() === 'uri'); + const usersIdentifiedByEmail = allGrantHeaders.filter( + it => it && it.userIDType.toLowerCase() === 'emailaddress', + ); + const usersIdentifiedByGroup = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'uri'); const justEmails = usersIdentifiedByEmail.map(item => item.identifier); - const validGroups = [ - constants.allAuthedUsersId, - constants.publicId, - constants.logId, - ]; + const validGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId]; for (let i = 0; i < usersIdentifiedByGroup.length; i++) { if (validGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) { return cb(errors.InvalidArgument); } } - const usersIdentifiedByID = allGrantHeaders - .filter(item => item && item.userIDType.toLowerCase() === 'id'); + const usersIdentifiedByID = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'id'); // TODO: Consider whether want to verify with Vault // whether canonicalID is associated with existing // account before adding to ACL @@ -148,22 +147,22 @@ const acl = { if (err) { return cb(err); } - const reconstructedUsersIdentifiedByEmail = aclUtils. - reconstructUsersIdentifiedByEmail(results, - usersIdentifiedByEmail); + const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail( + results, + usersIdentifiedByEmail, + ); const allUsers = [].concat( reconstructedUsersIdentifiedByEmail, usersIdentifiedByGroup, - usersIdentifiedByID); - const revisedACL = - aclUtils.sortHeaderGrants(allUsers, resourceACL); + usersIdentifiedByID, + ); + const revisedACL = aclUtils.sortHeaderGrants(allUsers, resourceACL); return cb(null, revisedACL); }); } else { // If don't have to look up canonicalID's just sort grants // and add to bucket - const revisedACL = aclUtils - .sortHeaderGrants(allGrantHeaders, resourceACL); + const revisedACL = aclUtils.sortHeaderGrants(allGrantHeaders, resourceACL); return cb(null, revisedACL); } return undefined; @@ -171,4 +170,3 @@ const acl = { }; module.exports = acl; - diff --git a/lib/services.js b/lib/services.js index 97116329a4..5bc5036442 100644 --- a/lib/services.js +++ b/lib/services.js @@ -13,8 +13,7 @@ const constants = require('../constants'); const { config } = require('./Config'); const { data } = require('./data/wrapper'); const metadata = require('./metadata/wrapper'); -const { setObjectLockInformation } - = require('./api/apiUtils/object/objectLockHelpers'); +const { setObjectLockInformation } = require('./api/apiUtils/object/objectLockHelpers'); const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked'); const { parseTagFromQuery } = s3middleware.tagging; @@ -41,52 +40,53 @@ const services = { // (without special increase) // TODO: Consider implementing pagination like object listing // with respect to bucket listing so can go beyond 10000 - metadata.listObject(bucketUsers, { prefix, maxKeys: 10000 }, log, - (err, listResponse) => { - // If MD responds with NoSuchBucket, this means the - // hidden usersBucket has not yet been created for - // the domain. If this is the case, it means - // that no buckets in this domain have been created so - // it follows that this particular user has no buckets. - // So, the get service listing should not have any - // buckets to list. By returning an empty array, the - // getService API will just respond with the user info - // without listing any buckets. - if (err?.is?.NoSuchBucket) { - log.trace('no buckets found'); - // If we checked the old user bucket, that means we - // already checked the new user bucket. If neither the - // old user bucket or the new user bucket exist, no buckets - // have yet been created in the namespace so an empty - // listing should be returned - if (overrideUserbucket) { - return cb(null, [], splitter); - } - // Since there were no results from checking the - // new users bucket, we check the old users bucket - return this.getService(authInfo, request, log, - constants.oldSplitter, cb, oldUsersBucket); - } - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); + metadata.listObject(bucketUsers, { prefix, maxKeys: 10000 }, log, (err, listResponse) => { + // If MD responds with NoSuchBucket, this means the + // hidden usersBucket has not yet been created for + // the domain. If this is the case, it means + // that no buckets in this domain have been created so + // it follows that this particular user has no buckets. + // So, the get service listing should not have any + // buckets to list. By returning an empty array, the + // getService API will just respond with the user info + // without listing any buckets. + if (err?.is?.NoSuchBucket) { + log.trace('no buckets found'); + // If we checked the old user bucket, that means we + // already checked the new user bucket. If neither the + // old user bucket or the new user bucket exist, no buckets + // have yet been created in the namespace so an empty + // listing should be returned + if (overrideUserbucket) { + return cb(null, [], splitter); } - return cb(null, listResponse.Contents, splitter); - }); + // Since there were no results from checking the + // new users bucket, we check the old users bucket + return this.getService(authInfo, request, log, constants.oldSplitter, cb, oldUsersBucket); + } + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); + } + return cb(null, listResponse.Contents, splitter); + }); }, - /** - * Check that hashedStream.completedHash matches header contentMd5. - * @param {object} contentMD5 - content-md5 header - * @param {string} completedHash - hashed stream once completed - * @param {RequestLogger} log - the current request logger - * @return {boolean} - true if contentMD5 matches or is undefined, - * false otherwise - */ + /** + * Check that hashedStream.completedHash matches header contentMd5. + * @param {object} contentMD5 - content-md5 header + * @param {string} completedHash - hashed stream once completed + * @param {RequestLogger} log - the current request logger + * @return {boolean} - true if contentMD5 matches or is undefined, + * false otherwise + */ checkHashMatchMD5(contentMD5, completedHash, log) { if (contentMD5 && completedHash && contentMD5 !== completedHash) { - log.debug('contentMD5 and completedHash does not match', - { method: 'checkHashMatchMD5', completedHash, contentMD5 }); + log.debug('contentMD5 and completedHash does not match', { + method: 'checkHashMatchMD5', + completedHash, + contentMD5, + }); return false; } return true; @@ -103,15 +103,46 @@ const services = { * @return {function} executes callback with err or ETag as arguments */ metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) { - const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders, - contentType, cacheControl, contentDisposition, contentEncoding, - expires, multipart, headers, overrideMetadata, log, - lastModifiedDate, versioning, versionId, uploadId, - tagging, taggingCopy, replicationInfo, defaultRetention, - dataStoreName, creationTime, retentionMode, retentionDate, - legalHold, originOp, updateMicroVersionId, archive, oldReplayId, - deleteNullKey, amzStorageClass, overheadField, needOplogUpdate, - restoredEtag, bucketOwnerId } = params; + const { + objectKey, + authInfo, + size, + contentMD5, + checksum, + metaHeaders, + contentType, + cacheControl, + contentDisposition, + contentEncoding, + expires, + multipart, + headers, + overrideMetadata, + log, + lastModifiedDate, + versioning, + versionId, + uploadId, + tagging, + taggingCopy, + replicationInfo, + defaultRetention, + dataStoreName, + creationTime, + retentionMode, + retentionDate, + legalHold, + originOp, + updateMicroVersionId, + archive, + oldReplayId, + deleteNullKey, + amzStorageClass, + overheadField, + needOplogUpdate, + restoredEtag, + bucketOwnerId, + } = params; log.trace('storing object in metadata'); assert.strictEqual(typeof bucketName, 'string'); const md = new ObjectMD(); @@ -196,12 +227,15 @@ const services = { // update restore if (archive) { md.setAmzStorageClass(amzStorageClass); - md.setArchive(new ObjectMDArchive( - archive.archiveInfo, - archive.restoreRequestedAt, - archive.restoreRequestedDays, - archive.restoreCompletedAt, - archive.restoreWillExpireAt)); + md.setArchive( + new ObjectMDArchive( + archive.archiveInfo, + archive.restoreRequestedAt, + archive.restoreRequestedDays, + archive.restoreCompletedAt, + archive.restoreWillExpireAt, + ), + ); md.setAmzRestore({ 'ongoing-request': false, 'expiry-date': archive.restoreWillExpireAt, @@ -287,55 +321,57 @@ const services = { // If this is not the completion of a multipart upload or // the creation of a delete marker, parse the headers to // get the ACL's if any - return async.waterfall([ - callback => { - if (multipart || md.getIsDeleteMarker()) { - return callback(); + return async.waterfall( + [ + callback => { + if (multipart || md.getIsDeleteMarker()) { + return callback(); + } + const parseAclParams = { + headers, + resourceType: 'object', + acl: md.getAcl(), + log, + }; + log.trace('parsing acl from headers'); + acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => { + if (err) { + log.debug('error parsing acl', { error: err }); + return callback(err); + } + md.setAcl(parsedACL); + return callback(); + }); + return null; + }, + callback => metadata.putObjectMD(bucketName, objectKey, md, options, log, callback), + ], + (err, data) => { + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); } - const parseAclParams = { - headers, - resourceType: 'object', - acl: md.getAcl(), - log, - }; - log.trace('parsing acl from headers'); - acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => { - if (err) { - log.debug('error parsing acl', { error: err }); - return callback(err); + log.trace('object successfully stored in metadata'); + // if versioning is enabled, data will be returned from metadata + // as JSON containing a versionId which some APIs will need sent + // back to them + let versionId; + if (data) { + if (params.isNull && params.isDeleteMarker) { + versionId = 'null'; + } else if (!params.isNull) { + versionId = JSON.parse(data).versionId; } - md.setAcl(parsedACL); - return callback(); + } + return cb(err, { + lastModified: md.getLastModified(), + tags: md.getTags(), + contentMD5, + versionId, + checksum: md.getChecksum(), }); - return null; }, - callback => metadata.putObjectMD(bucketName, objectKey, md, - options, log, callback), - ], (err, data) => { - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); - } - log.trace('object successfully stored in metadata'); - // if versioning is enabled, data will be returned from metadata - // as JSON containing a versionId which some APIs will need sent - // back to them - let versionId; - if (data) { - if (params.isNull && params.isDeleteMarker) { - versionId = 'null'; - } else if (!params.isNull) { - versionId = JSON.parse(data).versionId; - } - } - return cb(err, { - lastModified: md.getLastModified(), - tags: md.getTags(), - contentMD5, - versionId, - checksum: md.getChecksum(), - }); - }); + ); }, /** @@ -358,7 +394,11 @@ const services = { assert.strictEqual(typeof objectMD, 'object'); function deleteMDandData() { - return metadata.deleteObjectMD(bucketName, objectKey, options, log, + return metadata.deleteObjectMD( + bucketName, + objectKey, + options, + log, (err, res) => { if (err) { return cb(err, res); @@ -369,8 +409,7 @@ const services = { } if (deferLocationDeletion) { - return cb(null, Array.isArray(objectMD.location) - ? objectMD.location : [objectMD.location]); + return cb(null, Array.isArray(objectMD.location) ? objectMD.location : [objectMD.location]); } if (!Array.isArray(objectMD.location)) { @@ -384,14 +423,15 @@ const services = { } return cb(null, res); }); - }, originOp); + }, + originOp, + ); } const objGetInfo = objectMD.location; // special case that prevents azure blocks from unecessary deletion // will return null if no need - return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, - log, err => { + return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, log, err => { if (err) { return cb(err); } @@ -411,16 +451,14 @@ const services = { */ getObjectListing(bucketName, listingParams, log, cb) { assert.strictEqual(typeof bucketName, 'string'); - log.trace('performing metadata get object listing', - { listingParams }); - metadata.listObject(bucketName, listingParams, log, - (err, listResponse) => { - if (err) { - log.debug('error from metadata', { error: err }); - return cb(err); - } - return cb(null, listResponse); - }); + log.trace('performing metadata get object listing', { listingParams }); + metadata.listObject(bucketName, listingParams, log, (err, listResponse) => { + if (err) { + log.debug('error from metadata', { error: err }); + return cb(err); + } + return cb(null, listResponse); + }); }, /** @@ -463,10 +501,8 @@ const services = { } // Check each version in current batch for matching uploadId - const matchedVersion = (listResponse.Versions || []).find(version => - version.key === objectKey && - version.value && - version.value.uploadId === uploadId + const matchedVersion = (listResponse.Versions || []).find( + version => version.key === objectKey && version.value && version.value.uploadId === uploadId, ); if (matchedVersion) { @@ -484,7 +520,7 @@ const services = { return callback(); }); }, - err => cb(err, err ? null : foundVersion) + err => cb(err, err ? null : foundVersion), ); }, @@ -500,16 +536,14 @@ const services = { */ getLifecycleListing(bucketName, listingParams, log, cb) { assert.strictEqual(typeof bucketName, 'string'); - log.trace('performing metadata get object listing for lifecycle', - { listingParams }); - metadata.listLifecycleObject(bucketName, listingParams, log, - (err, listResponse) => { - if (err) { - log.debug('error from metadata', { error: err }); - return cb(err); - } - return cb(null, listResponse); - }); + log.trace('performing metadata get object listing for lifecycle', { listingParams }); + metadata.listLifecycleObject(bucketName, listingParams, log, (err, listResponse) => { + if (err) { + log.debug('error from metadata', { error: err }); + return cb(err); + } + return cb(null, listResponse); + }); }, metadataStoreMPObject(bucketName, cipherBundle, params, log, cb) { @@ -522,9 +556,7 @@ const services = { // the splitter. // 2) UploadId's are UUID version 4 const splitter = params.splitter; - const longMPUIdentifier = - `overview${splitter}${params.objectKey}` + - `${splitter}${params.uploadId}`; + const longMPUIdentifier = `overview${splitter}${params.objectKey}` + `${splitter}${params.uploadId}`; const multipartObjectMD = {}; multipartObjectMD.id = params.uploadId; multipartObjectMD.eventualStorageBucket = params.eventualStorageBucket; @@ -546,28 +578,19 @@ const services = { multipartObjectMD.key = params.objectKey; multipartObjectMD.uploadId = params.uploadId; multipartObjectMD['cache-control'] = params.headers['cache-control']; - multipartObjectMD['content-disposition'] = - params.headers['content-disposition']; - multipartObjectMD['content-encoding'] = - removeAWSChunked(params.headers['content-encoding']); - multipartObjectMD['content-type'] = - params.headers['content-type']; - multipartObjectMD.expires = - params.headers.expires; - multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639 - multipartObjectMD['x-amz-website-redirect-location'] = - params.headers['x-amz-website-redirect-location']; + multipartObjectMD['content-disposition'] = params.headers['content-disposition']; + multipartObjectMD['content-encoding'] = removeAWSChunked(params.headers['content-encoding']); + multipartObjectMD['content-type'] = params.headers['content-type']; + multipartObjectMD.expires = params.headers.expires; + multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639 + multipartObjectMD['x-amz-website-redirect-location'] = params.headers['x-amz-website-redirect-location']; if (cipherBundle) { - multipartObjectMD['x-amz-server-side-encryption'] = - cipherBundle.algorithm; + multipartObjectMD['x-amz-server-side-encryption'] = cipherBundle.algorithm; if (cipherBundle.masterKeyId) { - multipartObjectMD[ - 'x-amz-server-side-encryption-aws-kms-key-id'] = - cipherBundle.masterKeyId; + multipartObjectMD['x-amz-server-side-encryption-aws-kms-key-id'] = cipherBundle.masterKeyId; } } - multipartObjectMD.controllingLocationConstraint = - params.controllingLocationConstraint; + multipartObjectMD.controllingLocationConstraint = params.controllingLocationConstraint; multipartObjectMD.dataStoreName = params.dataStoreName; if (params.tagging) { const validationTagRes = parseTagFromQuery(params.tagging); @@ -613,15 +636,14 @@ const services = { return cb(err); } multipartObjectMD.acl = parsedACL; - metadata.putObjectMD(bucketName, longMPUIdentifier, - multipartObjectMD, {}, log, err => { - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); - } + metadata.putObjectMD(bucketName, longMPUIdentifier, multipartObjectMD, {}, log, err => { + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); + } - return cb(null, multipartObjectMD); - }); + return cb(null, multipartObjectMD); + }); return undefined; }); }, @@ -648,12 +670,10 @@ const services = { assert.strictEqual(typeof params.splitter, 'string'); assert.strictEqual(typeof params.storedMetadata, 'object'); const splitter = params.splitter; - const longMPUIdentifier = - `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; + const longMPUIdentifier = `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; const multipartObjectMD = Object.assign({}, params.storedMetadata); multipartObjectMD.completeInProgress = true; - metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD, - {}, log, err => { + metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD, {}, log, err => { if (err) { log.error('error from metadata', { error: err }); return cb(err); @@ -685,30 +705,28 @@ const services = { const mpuBucketName = `${constants.mpuBucketPrefix}${params.bucketName}`; const splitter = params.splitter; - const mpuOverviewKey = - `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; - return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, - (err, res) => { - if (err) { - if (err.is && err.is.NoSuchKey) { - // The overview key no longer exists, meaning completeMultipartUpload - // already ran to completion and cleaned up the MPU bucket. - // This is a race condition: objectPutPart checked for old - // part locations after completeMultipartUpload deleted the overview. - // Returning true (complete in progress) prevents objectPutPart - // from deleting part data that may have already been committed - // as the final object. - return cb(null, true); - } - log.error('error getting the overview object from mpu bucket', { - error: err, - method: 'services.isCompleteMPUInProgress', - params, - }); - return cb(err); + const mpuOverviewKey = `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`; + return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => { + if (err) { + if (err.is && err.is.NoSuchKey) { + // The overview key no longer exists, meaning completeMultipartUpload + // already ran to completion and cleaned up the MPU bucket. + // This is a race condition: objectPutPart checked for old + // part locations after completeMultipartUpload deleted the overview. + // Returning true (complete in progress) prevents objectPutPart + // from deleting part data that may have already been committed + // as the final object. + return cb(null, true); } - return cb(null, Boolean(res.completeInProgress)); - }); + log.error('error getting the overview object from mpu bucket', { + error: err, + method: 'services.isCompleteMPUInProgress', + params, + }); + return cb(err); + } + return cb(null, Boolean(res.completeInProgress)); + }); }, /** @@ -725,8 +743,7 @@ const services = { * - the overview key stored metadata */ metadataValidateMultipart(params, cb) { - const { bucketName, uploadId, authInfo, - objectKey, requestType, log } = params; + const { bucketName, uploadId, authInfo, objectKey, requestType, log } = params; assert.strictEqual(typeof bucketName, 'string'); // This checks whether the mpu bucket exists. @@ -734,13 +751,11 @@ const services = { const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`; metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => { if (err?.is?.NoSuchBucket) { - log.debug('bucket not found in metadata', { error: err, - method: 'services.metadataValidateMultipart' }); + log.debug('bucket not found in metadata', { error: err, method: 'services.metadataValidateMultipart' }); return cb(errors.NoSuchUpload); } if (err) { - log.error('error from metadata', { error: err, - method: 'services.metadataValidateMultipart' }); + log.error('error from metadata', { error: err, method: 'services.metadataValidateMultipart' }); return cb(err); } @@ -749,87 +764,78 @@ const services = { if (mpuBucket.getMdBucketModelVersion() < 2) { splitter = constants.oldSplitter; } - const mpuOverviewKey = - `overview${splitter}${objectKey}${splitter}${uploadId}`; + const mpuOverviewKey = `overview${splitter}${objectKey}${splitter}${uploadId}`; - metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, - {}, log, (err, storedMetadata) => { - if (err) { - if (err.is && err.is.NoSuchKey) { - return cb(errors.NoSuchUpload); - } - log.error('error from metadata', { error: err }); - return cb(err); + metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, {}, log, (err, storedMetadata) => { + if (err) { + if (err.is && err.is.NoSuchKey) { + return cb(errors.NoSuchUpload); } + log.error('error from metadata', { error: err }); + return cb(err); + } - const initiatorID = storedMetadata.initiator.ID; - const ownerID = storedMetadata['owner-id']; - const mpuOverview = { - key: storedMetadata.key, - id: storedMetadata.id, - eventualStorageBucket: - storedMetadata.eventualStorageBucket, - initiatorID, - initiatorDisplayName: - storedMetadata.initiator.DisplayName, - ownerID, - ownerDisplayName: - storedMetadata['owner-display-name'], - storageClass: - storedMetadata['x-amz-storage-class'], - initiated: storedMetadata.initiated, - controllingLocationConstraint: - storedMetadata.controllingLocationConstraint, - checksumAlgorithm: storedMetadata.checksumAlgorithm, - checksumType: storedMetadata.checksumType, - checksumIsDefault: storedMetadata.checksumIsDefault, - }; + const initiatorID = storedMetadata.initiator.ID; + const ownerID = storedMetadata['owner-id']; + const mpuOverview = { + key: storedMetadata.key, + id: storedMetadata.id, + eventualStorageBucket: storedMetadata.eventualStorageBucket, + initiatorID, + initiatorDisplayName: storedMetadata.initiator.DisplayName, + ownerID, + ownerDisplayName: storedMetadata['owner-display-name'], + storageClass: storedMetadata['x-amz-storage-class'], + initiated: storedMetadata.initiated, + controllingLocationConstraint: storedMetadata.controllingLocationConstraint, + checksumAlgorithm: storedMetadata.checksumAlgorithm, + checksumType: storedMetadata.checksumType, + checksumIsDefault: storedMetadata.checksumIsDefault, + }; - const tagging = storedMetadata['x-amz-tagging']; - if (tagging) { - mpuOverview.tagging = tagging; - } - // If access was provided by the destination bucket's - // bucket policies, go ahead. - if (requestType === 'bucketPolicyGoAhead') { - return cb(null, mpuBucket, mpuOverview, storedMetadata); - } + const tagging = storedMetadata['x-amz-tagging']; + if (tagging) { + mpuOverview.tagging = tagging; + } + // If access was provided by the destination bucket's + // bucket policies, go ahead. + if (requestType === 'bucketPolicyGoAhead') { + return cb(null, mpuBucket, mpuOverview, storedMetadata); + } - const requesterID = authInfo.isRequesterAnIAMUser() ? - authInfo.getArn() : authInfo.getCanonicalID(); - const isRequesterInitiator = - initiatorID === requesterID; - const isRequesterParentAccountOfInitiator = - ownerID === authInfo.getCanonicalID(); - if (requestType === 'putPart or complete') { - // Only the initiator of the multipart - // upload can upload a part or complete the mpu - if (!isRequesterInitiator) { - return cb(errors.AccessDenied); - } + const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID(); + const isRequesterInitiator = initiatorID === requesterID; + const isRequesterParentAccountOfInitiator = ownerID === authInfo.getCanonicalID(); + if (requestType === 'putPart or complete') { + // Only the initiator of the multipart + // upload can upload a part or complete the mpu + if (!isRequesterInitiator) { + return cb(errors.AccessDenied); } - if (requestType === 'deleteMPU' - || requestType === 'listParts') { - // In order for account/user to be - // authorized must either be the - // bucket owner or intitator of - // the multipart upload request - // (or parent account of initiator). - // In addition if the bucket policy - // designates someone else with - // s3:AbortMultipartUpload or - // s3:ListMultipartUploadPartsrights, - // as applicable, that account/user will have the right. - // If got to this step, it means there is - // no bucket policy on this. - if (mpuBucket.getOwner() !== authInfo.getCanonicalID() - && !isRequesterInitiator - && !isRequesterParentAccountOfInitiator) { - return cb(errors.AccessDenied); - } + } + if (requestType === 'deleteMPU' || requestType === 'listParts') { + // In order for account/user to be + // authorized must either be the + // bucket owner or intitator of + // the multipart upload request + // (or parent account of initiator). + // In addition if the bucket policy + // designates someone else with + // s3:AbortMultipartUpload or + // s3:ListMultipartUploadPartsrights, + // as applicable, that account/user will have the right. + // If got to this step, it means there is + // no bucket policy on this. + if ( + mpuBucket.getOwner() !== authInfo.getCanonicalID() && + !isRequesterInitiator && + !isRequesterParentAccountOfInitiator + ) { + return cb(errors.AccessDenied); } - return cb(null, mpuBucket, mpuOverview, storedMetadata); - }); + } + return cb(null, mpuBucket, mpuOverview, storedMetadata); + }); return undefined; }); }, @@ -851,13 +857,11 @@ const services = { * @param {function} cb - callback to send error or move to next task * @return {undefined} */ - metadataStorePart(mpuBucketName, partLocations, - metaStoreParams, log, cb) { + metadataStorePart(mpuBucketName, partLocations, metaStoreParams, log, cb) { assert.strictEqual(typeof mpuBucketName, 'string'); - const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId } - = metaStoreParams; - const dateModified = typeof lastModified === 'string' ? - lastModified : new Date().toJSON(); + const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId } = + metaStoreParams; + const dateModified = typeof lastModified === 'string' ? lastModified : new Date().toJSON(); assert.strictEqual(typeof splitter, 'string'); const partKey = `${uploadId}${splitter}${partNumber}`; const omVal = { @@ -865,7 +869,7 @@ const services = { // from an object to an array 'md-model-version': 3, partLocations, - 'key': partKey, + key: partKey, 'last-modified': dateModified, 'content-md5': contentMD5, 'content-length': size, @@ -887,14 +891,14 @@ const services = { }, /** - * Gets list of open multipart uploads in bucket - * @param {object} MPUbucketName - bucket in which objectMetadata is stored - * @param {object} listingParams - params object passing on - * needed items from request object - * @param {object} log - Werelogs logger - * @param {function} cb - callback to listMultipartUploads.js - * @return {undefined} - */ + * Gets list of open multipart uploads in bucket + * @param {object} MPUbucketName - bucket in which objectMetadata is stored + * @param {object} listingParams - params object passing on + * needed items from request object + * @param {object} log - Werelogs logger + * @param {function} cb - callback to listMultipartUploads.js + * @return {undefined} + */ getMultipartUploadListing(MPUbucketName, listingParams, log, cb) { assert.strictEqual(typeof MPUbucketName, 'string'); assert.strictEqual(typeof listingParams.splitter, 'string'); @@ -921,8 +925,7 @@ const services = { if (bucket.getMdBucketModelVersion() < 2) { listParams.splitter = constants.oldSplitter; } - metadata.listMultipartUploads(MPUbucketName, listParams, log, - cb); + metadata.listMultipartUploads(MPUbucketName, listParams, log, cb); return undefined; }); }, @@ -944,24 +947,26 @@ const services = { if (err?.is?.NoSuchBucket) { log.trace('no buckets found'); const creationDate = new Date().toJSON(); - const mpuBucket = new BucketInfo(MPUBucketName, + const mpuBucket = new BucketInfo( + MPUBucketName, destinationBucket.getOwner(), - destinationBucket.getOwnerDisplayName(), creationDate, - BucketInfo.currentModelVersion()); + destinationBucket.getOwnerDisplayName(), + creationDate, + BucketInfo.currentModelVersion(), + ); // Note that unlike during the creation of a normal bucket, // we do NOT add this bucket to the lists of a user's buckets. // By not adding this bucket to the lists of a user's buckets, // a getService request should not return a reference to this // bucket. This is the desired behavior since this should be // a hidden bucket. - return metadata.createBucket(MPUBucketName, mpuBucket, log, - err => { - if (err) { - log.error('error from metadata', { error: err }); - return cb(err); - } - return cb(null, mpuBucket); - }); + return metadata.createBucket(MPUBucketName, mpuBucket, log, err => { + if (err) { + log.error('error from metadata', { error: err }); + return cb(err); + } + return cb(null, mpuBucket); + }); } if (err) { log.error('error from metadata', { @@ -986,8 +991,7 @@ const services = { }, getSomeMPUparts(params, cb) { - const { uploadId, mpuBucketName, maxParts, partNumberMarker, log } = - params; + const { uploadId, mpuBucketName, maxParts, partNumberMarker, log } = params; assert.strictEqual(typeof mpuBucketName, 'string'); assert.strictEqual(typeof params.splitter, 'string'); const paddedPartNumber = `000000${partNumberMarker}`.substr(-5); @@ -1004,9 +1008,14 @@ const services = { // If have efficient way to batch delete metadata, should so this // all at once in production implementation assert.strictEqual(typeof mpuBucketName, 'string'); - async.eachLimit(keysToDelete, 5, (key, callback) => { - metadata.deleteObjectMD(mpuBucketName, key, { overheadField: constants.overheadField }, log, callback); - }, cb); + async.eachLimit( + keysToDelete, + 5, + (key, callback) => { + metadata.deleteObjectMD(mpuBucketName, key, { overheadField: constants.overheadField }, log, callback); + }, + cb, + ); }, }; diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index a754300c62..a8136cfe25 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -1,6 +1,5 @@ const { getVersionIdResHeader } = require('../api/apiUtils/object/versioning'); -const checkUserMetadataSize - = require('../api/apiUtils/object/checkUserMetadataSize'); +const checkUserMetadataSize = require('../api/apiUtils/object/checkUserMetadataSize'); const { getAmzRestoreResHeader } = require('../api/apiUtils/object/coldStorage'); const { config } = require('../Config'); const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface'); @@ -16,38 +15,36 @@ const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface'); * @return {object} responseMetaHeaders headers with object metadata to include * in response to client */ -function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, - returnTagCount) { +function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagCount) { // Add user meta headers from objectMD let responseMetaHeaders = Object.assign({}, corsHeaders); - Object.keys(objectMD).filter(val => (val.startsWith('x-amz-meta-'))) - .forEach(id => { responseMetaHeaders[id] = objectMD[id]; }); + Object.keys(objectMD) + .filter(val => val.startsWith('x-amz-meta-')) + .forEach(id => { + responseMetaHeaders[id] = objectMD[id]; + }); // Check user metadata size responseMetaHeaders = checkUserMetadataSize(responseMetaHeaders); // TODO: When implement lifecycle, add additional response headers // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html - responseMetaHeaders['x-amz-version-id'] = - getVersionIdResHeader(versioningCfg, objectMD); + responseMetaHeaders['x-amz-version-id'] = getVersionIdResHeader(versioningCfg, objectMD); if (objectMD['x-amz-website-redirect-location']) { - responseMetaHeaders['x-amz-website-redirect-location'] = - objectMD['x-amz-website-redirect-location']; + responseMetaHeaders['x-amz-website-redirect-location'] = objectMD['x-amz-website-redirect-location']; } if (objectMD['x-amz-storage-class'] !== 'STANDARD') { - responseMetaHeaders['x-amz-storage-class'] = - objectMD['x-amz-storage-class']; + responseMetaHeaders['x-amz-storage-class'] = objectMD['x-amz-storage-class']; } if (objectMD['x-amz-server-side-encryption']) { - responseMetaHeaders['x-amz-server-side-encryption'] - = objectMD['x-amz-server-side-encryption']; + responseMetaHeaders['x-amz-server-side-encryption'] = objectMD['x-amz-server-side-encryption']; } const kmsKey = objectMD['x-amz-server-side-encryption-aws-kms-key-id']; - if (kmsKey && - objectMD['x-amz-server-side-encryption'] === 'aws:kms') { - responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] - = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKey) : kmsKey; + if (kmsKey && objectMD['x-amz-server-side-encryption'] === 'aws:kms') { + responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] = config.kmsHideScalityArn + ? getKeyIdFromArn(kmsKey) + : kmsKey; } const restoreHeader = getAmzRestoreResHeader(objectMD); @@ -65,8 +62,7 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, responseMetaHeaders['Cache-Control'] = objectMD['cache-control']; } if (objectMD['content-disposition']) { - responseMetaHeaders['Content-Disposition'] - = objectMD['content-disposition']; + responseMetaHeaders['Content-Disposition'] = objectMD['content-disposition']; } if (objectMD['content-encoding']) { responseMetaHeaders['Content-Encoding'] = objectMD['content-encoding']; @@ -78,40 +74,30 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, // Note: ETag must have a capital "E" and capital "T" for cosbench // to work. responseMetaHeaders.ETag = `"${objectMD['content-md5']}"`; - responseMetaHeaders['Last-Modified'] = - new Date(objectMD['last-modified']).toUTCString(); + responseMetaHeaders['Last-Modified'] = new Date(objectMD['last-modified']).toUTCString(); if (objectMD['content-type']) { responseMetaHeaders['Content-Type'] = objectMD['content-type']; } - if (returnTagCount && objectMD.tags && - Object.keys(objectMD.tags).length > 0) { - responseMetaHeaders['x-amz-tagging-count'] = - Object.keys(objectMD.tags).length; + if (returnTagCount && objectMD.tags && Object.keys(objectMD.tags).length > 0) { + responseMetaHeaders['x-amz-tagging-count'] = Object.keys(objectMD.tags).length; } - const hasRetentionInfo = objectMD.retentionMode - && objectMD.retentionDate; + const hasRetentionInfo = objectMD.retentionMode && objectMD.retentionDate; if (hasRetentionInfo) { - responseMetaHeaders['x-amz-object-lock-retain-until-date'] - = objectMD.retentionDate; - responseMetaHeaders['x-amz-object-lock-mode'] - = objectMD.retentionMode; + responseMetaHeaders['x-amz-object-lock-retain-until-date'] = objectMD.retentionDate; + responseMetaHeaders['x-amz-object-lock-mode'] = objectMD.retentionMode; } if (objectMD.legalHold !== undefined) { - responseMetaHeaders['x-amz-object-lock-legal-hold'] - = objectMD.legalHold ? 'ON' : 'OFF'; + responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF'; } if (objectMD.replicationInfo && objectMD.replicationInfo.status) { - responseMetaHeaders['x-amz-replication-status'] = - objectMD.replicationInfo.status; + responseMetaHeaders['x-amz-replication-status'] = objectMD.replicationInfo.status; } if (Array.isArray(objectMD?.replicationInfo?.backends)) { objectMD.replicationInfo.backends.forEach(backend => { const { status, site, dataStoreVersionId } = backend; - responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = - status; + responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = status; if (status === 'COMPLETED' && dataStoreVersionId) { - responseMetaHeaders[`x-amz-meta-${site}-version-id`] = - dataStoreVersionId; + responseMetaHeaders[`x-amz-meta-${site}-version-id`] = dataStoreVersionId; } }); } diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index 48451b43ce..fa228cf3cb 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -4,16 +4,14 @@ const crypto = require('crypto'); const BucketInfo = require('arsenal').models.BucketInfo; -const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = - require('../helpers'); +const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers'); const constants = require('../../../constants'); const { metadata } = require('arsenal').storage.metadata.inMemory.metadata; const DummyRequest = require('../DummyRequest'); const { objectDelete } = require('../../../lib/api/objectDelete'); const objectPut = require('../../../lib/api/objectPut'); const objectCopy = require('../../../lib/api/objectCopy'); -const completeMultipartUpload = - require('../../../lib/api/completeMultipartUpload'); +const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload'); const objectPutACL = require('../../../lib/api/objectPutACL'); const objectPutTagging = require('../../../lib/api/objectPutTagging'); const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging'); @@ -55,19 +53,20 @@ const objectACLReq = { // Get an object request with the given key. function getObjectPutReq(key, hasContent) { const bodyContent = hasContent ? 'body content' : ''; - return new DummyRequest({ - bucketName, - namespace, - objectKey: key, - headers: {}, - url: `/${bucketName}/${key}`, - }, Buffer.from(bodyContent, 'utf8')); + return new DummyRequest( + { + bucketName, + namespace, + objectKey: key, + headers: {}, + url: `/${bucketName}/${key}`, + }, + Buffer.from(bodyContent, 'utf8'), + ); } -const taggingPutReq = new TaggingConfigTester() - .createObjectTaggingRequest('PUT', bucketName, keyA); -const taggingDeleteReq = new TaggingConfigTester() - .createObjectTaggingRequest('DELETE', bucketName, keyA); +const taggingPutReq = new TaggingConfigTester().createObjectTaggingRequest('PUT', bucketName, keyA); +const taggingDeleteReq = new TaggingConfigTester().createObjectTaggingRequest('DELETE', bucketName, keyA); const emptyReplicationMD = { status: '', @@ -99,35 +98,34 @@ function checkObjectReplicationInfo(key, expected) { // Put the object key and check the replication information. function putObjectAndCheckMD(key, expected, cb) { - return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, - err => { - if (err) { - return cb(err); - } - checkObjectReplicationInfo(key, expected); - return cb(); - }); + return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, err => { + if (err) { + return cb(err); + } + checkObjectReplicationInfo(key, expected); + return cb(); + }); } // Create the bucket in metadata. function createBucket() { - metadata - .buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', '')); - metadata.keyMaps.set(bucketName, new Map); + metadata.buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', '')); + metadata.keyMaps.set(bucketName, new Map()); } // Create the bucket in metadata with versioning and a replication config. function createBucketWithReplication(hasStorageClass) { createBucket(); const config = { - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', destination: 'arn:aws:s3:::source-bucket', - rules: [{ - prefix: keyA, - enabled: true, - id: 'test-id', - }], + rules: [ + { + prefix: keyA, + enabled: true, + id: 'test-id', + }, + ], }; if (hasStorageClass) { config.rules[0].storageClass = storageClassType; @@ -140,22 +138,21 @@ function createBucketWithReplication(hasStorageClass) { // Create the shadow bucket in metadata for MPUs with a recent model number. function createShadowBucket(key, uploadId) { - const overviewKey = `overview${constants.splitter}` + - `${key}${constants.splitter}${uploadId}`; - metadata.buckets - .set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', '')); - // Set modelVersion to use the most recent splitter. + const overviewKey = `overview${constants.splitter}` + `${key}${constants.splitter}${uploadId}`; + metadata.buckets.set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', '')); + // Set modelVersion to use the most recent splitter. Object.assign(metadata.buckets.get(mpuShadowBucket), { _mdBucketModelVersion: 5, }); - metadata.keyMaps.set(mpuShadowBucket, new Map); - metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map); + metadata.keyMaps.set(mpuShadowBucket, new Map()); + metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map()); Object.assign(metadata.keyMaps.get(mpuShadowBucket).get(overviewKey), { id: uploadId, eventualStorageBucket: bucketName, initiator: { DisplayName: 'accessKey1displayName', - ID: ownerID }, + ID: ownerID, + }, key, uploadId, }); @@ -170,24 +167,26 @@ function putMPU(key, body, cb) { const calculatedHash = md5Hash.digest('hex'); const partKey = `${uploadId}${constants.splitter}00001`; const obj = { - partLocations: [{ - key: 1, - dataStoreName: 'scality-internal-mem', - dataStoreETag: `1:${calculatedHash}`, - }], + partLocations: [ + { + key: 1, + dataStoreName: 'scality-internal-mem', + dataStoreETag: `1:${calculatedHash}`, + }, + ], key: partKey, }; obj['content-md5'] = calculatedHash; obj['content-length'] = body.length; - metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map); + metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map()); const partMap = metadata.keyMaps.get(mpuShadowBucket).get(partKey); Object.assign(partMap, obj); const postBody = '' + - '' + - '1' + - `"${calculatedHash}"` + - '' + + '' + + '1' + + `"${calculatedHash}"` + + '' + ''; const req = { bucketName, @@ -217,8 +216,7 @@ function copyObject(sourceObjectKey, copyObjectKey, hasContent, cb) { headers: {}, url: `/${bucketName}/${sourceObjectKey}`, }); - return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, - log, cb); + return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, log, cb); }); } @@ -230,26 +228,33 @@ describe('Replication object MD without bucket replication config', () => { afterEach(() => cleanup()); - it('should not update object metadata', done => - putObjectAndCheckMD(keyA, emptyReplicationMD, done)); + it('should not update object metadata', done => putObjectAndCheckMD(keyA, emptyReplicationMD, done)); it('should not update object metadata if putting object ACL', done => - async.series([ - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectPutACL(authInfo, objectACLReq, log, next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); - return done(); - })); + async.series( + [ + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectPutACL(authInfo, objectACLReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + return done(); + }, + )); describe('Object tagging', () => { - beforeEach(done => async.series([ - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectPutTagging(authInfo, taggingPutReq, log, next), - ], err => done(err))); + beforeEach(done => + async.series( + [ + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => done(err), + ), + ); it('should not update object metadata if putting tag', done => { checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); @@ -257,18 +262,20 @@ describe('Replication object MD without bucket replication config', () => { }); it('should not update object metadata if deleting tag', done => - async.series([ - // Put a new version to update replication MD content array. - next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, log, - next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); - return done(); - })); + async.series( + [ + // Put a new version to update replication MD content array. + next => putObjectAndCheckMD(keyA, emptyReplicationMD, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD); + return done(); + }, + )); it('should not update object metadata if completing MPU', done => putMPU(keyA, 'content', err => { @@ -291,430 +298,455 @@ describe('Replication object MD without bucket replication config', () => { }); [true, false].forEach(hasStorageClass => { - describe('Replication object MD with bucket replication config ' + - `${hasStorageClass ? 'with' : 'without'} storage class`, () => { - const replicationMD = { - status: 'PENDING', - backends: [{ - site: 'zenko', + describe( + 'Replication object MD with bucket replication config ' + + `${hasStorageClass ? 'with' : 'without'} storage class`, + () => { + const replicationMD = { status: 'PENDING', + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: '', + }, + ], + content: ['DATA', 'METADATA'], + destination: bucketARN, + storageClass: 'zenko', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + storageType: '', dataStoreVersionId: '', - }], - content: ['DATA', 'METADATA'], - destination: bucketARN, - storageClass: 'zenko', - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - storageType: '', - dataStoreVersionId: '', - isNFS: undefined, - }; - const newReplicationMD = hasStorageClass ? Object.assign(replicationMD, - { storageClass: storageClassType }) : replicationMD; - const replicateMetadataOnly = Object.assign({}, newReplicationMD, - { content: ['METADATA'] }); - - beforeEach(() => { - cleanup(); - createBucketWithReplication(hasStorageClass); - }); - - afterEach(() => { - cleanup(); - delete config.locationConstraints['zenko']; - }); - - it('should update metadata when replication config prefix matches ' + - 'an object key', done => - putObjectAndCheckMD(keyA, newReplicationMD, done)); - - it('should update metadata when replication config prefix matches ' + - 'the start of an object key', done => - putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done)); - - it('should not update metadata when replication config prefix does ' + - 'not match the start of an object key', done => - putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done)); - - it('should not update metadata when replication config prefix does ' + - 'not apply', done => - putObjectAndCheckMD(keyB, emptyReplicationMD, done)); - - it("should update status to 'PENDING' if putting a new version", done => - putObjectAndCheckMD(keyA, newReplicationMD, err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - return putObjectAndCheckMD(keyA, newReplicationMD, done); - })); - - it("should update status to 'PENDING' and content to '['METADATA']' " + - 'if putting 0 byte object', done => - objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, - err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - - it('should update metadata if putting object ACL and CRR replication', done => { - // Set 'zenko' as a typical CRR location (i.e. no type) - config.locationConstraints['zenko'] = { - ...config.locationConstraints['zenko'], - type: '', + isNFS: undefined, }; - - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => { - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); + const newReplicationMD = hasStorageClass + ? Object.assign(replicationMD, { storageClass: storageClassType }) + : replicationMD; + const replicateMetadataOnly = Object.assign({}, newReplicationMD, { content: ['METADATA'] }); + + beforeEach(() => { + cleanup(); + createBucketWithReplication(hasStorageClass); }); - }); - - it('should not update metadata if putting object ACL and cloud replication', done => { - // Set 'zenko' as a typical cloud location (i.e. type) - config.locationConstraints['zenko'] = { - ...config.locationConstraints['zenko'], - type: 'aws_s3', - }; - const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; - - let completedReplicationInfo; - async.series([ - next => putObjectAndCheckMD(keyA, replicationMD, next), - next => { - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Update metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - completedReplicationInfo = JSON.parse( - JSON.stringify(objectMD.replicationInfo)); - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, completedReplicationInfo); - return done(); + afterEach(() => { + cleanup(); + delete config.locationConstraints['zenko']; }); - }); - - it('should update metadata if putting a delete marker', done => - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, err => { - if (err) { - return next(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - // Set metadata to a status after replication has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - return next(); - }), - next => objectDelete(authInfo, deleteReq, log, next), - ], err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - assert.strictEqual(objectMD.isDeleteMarker, true); - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - it('should not update metadata if putting a delete marker owned by ' + - 'Lifecycle service account', done => - async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectDelete(authInfoLifecycleService, deleteReq, - log, next), - ], err => { - if (err) { - return done(err); - } - const objectMD = metadata.keyMaps.get(bucketName).get(keyA); - assert.strictEqual(objectMD.isDeleteMarker, true); - checkObjectReplicationInfo(keyA, emptyReplicationMD); - return done(); - })); + it('should update metadata when replication config prefix matches ' + 'an object key', done => + putObjectAndCheckMD(keyA, newReplicationMD, done), + ); - describe('Object tagging', () => { - beforeEach(done => async.series([ - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectPutTagging(authInfo, taggingPutReq, log, next), - ], err => done(err))); + it('should update metadata when replication config prefix matches ' + 'the start of an object key', done => + putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']'if putting tag", done => { - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - }); + it( + 'should not update metadata when replication config prefix does ' + + 'not match the start of an object key', + done => putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if deleting tag", done => - async.series([ - // Put a new version to update replication MD content array. - next => putObjectAndCheckMD(keyA, newReplicationMD, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, log, - next), - ], err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); - }); + it('should not update metadata when replication config prefix does ' + 'not apply', done => + putObjectAndCheckMD(keyB, emptyReplicationMD, done), + ); - describe('Complete MPU', () => { - it("should update status to 'PENDING' and content to " + - "'['DATA, METADATA']' if completing MPU", done => - putMPU(keyA, 'content', err => { + it("should update status to 'PENDING' if putting a new version", done => + putObjectAndCheckMD(keyA, newReplicationMD, err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, newReplicationMD); - return done(); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + return putObjectAndCheckMD(keyA, newReplicationMD, done); })); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if completing MPU with 0 bytes", done => - putMPU(keyA, '', err => { + it("should update status to 'PENDING' and content to '['METADATA']' " + 'if putting 0 byte object', done => + objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, err => { if (err) { return done(err); } checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - })); - - it('should not update replicationInfo if key does not apply', - done => putMPU(keyB, 'content', err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyB, emptyReplicationMD); - return done(); - })); - }); - - describe('Object copy', () => { - it("should update status to 'PENDING' and content to " + - "'['DATA, METADATA']' if copying object", done => - copyObject(keyB, keyA, true, err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, newReplicationMD); - return done(); - })); + }), + ); - it("should update status to 'PENDING' and content to " + - "'['METADATA']' if copying object with 0 bytes", done => - copyObject(keyB, keyA, false, err => { - if (err) { - return done(err); - } - checkObjectReplicationInfo(keyA, replicateMetadataOnly); - return done(); - })); + it('should update metadata if putting object ACL and CRR replication', done => { + // Set 'zenko' as a typical CRR location (i.e. no type) + config.locationConstraints['zenko'] = { + ...config.locationConstraints['zenko'], + type: '', + }; - it('should not update replicationInfo if key does not apply', - done => { - const copyKey = `foo-${keyA}`; - return copyObject(keyB, copyKey, true, err => { + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(copyKey, emptyReplicationMD); + checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - }); - }); - }); + }, + ); + }); - ['awsbackend', - 'azurebackend', - 'gcpbackend', - 'awsbackend,azurebackend'].forEach(backend => { - const storageTypeMap = { - 'awsbackend': 'aws_s3', - 'azurebackend': 'azure', - 'gcpbackend': 'gcp', - 'awsbackend,azurebackend': 'aws_s3,azure', - }; - const storageType = storageTypeMap[backend]; - const backends = backend.split(',').map(site => ({ - site, - status: 'PENDING', - dataStoreVersionId: '', - })); - describe('Object metadata replicationInfo storageType value', - () => { - const expectedReplicationInfo = { - status: 'PENDING', - backends, - content: ['DATA', 'METADATA'], - destination: 'arn:aws:s3:::destination-bucket', - storageClass: backend, - role: 'arn:aws:iam::account-id:role/resource', - storageType, - dataStoreVersionId: '', - isNFS: undefined, + it('should not update metadata if putting object ACL and cloud replication', done => { + // Set 'zenko' as a typical cloud location (i.e. type) + config.locationConstraints['zenko'] = { + ...config.locationConstraints['zenko'], + type: 'aws_s3', }; - // Expected for a metadata-only replication operation (for - // example, putting object tags). - const expectedReplicationInfoMD = Object.assign({}, - expectedReplicationInfo, { content: ['METADATA'] }); - - beforeEach(() => - // We have already created the bucket, so update the - // replication configuration to include a location - // constraint for the `storageClass`. This results in a - // `storageType` of 'aws_s3', for example. - Object.assign(metadata.buckets.get(bucketName), { - _replicationConfiguration: { - role: 'arn:aws:iam::account-id:role/resource', - destination: 'arn:aws:s3:::destination-bucket', - rules: [{ - prefix: keyA, - enabled: true, - id: 'test-id', - storageClass: backend, - }], - }, - })); + const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' }; - it('should update on a put object request', done => - putObjectAndCheckMD(keyA, expectedReplicationInfo, done)); - - it('should update on a complete MPU object request', done => - putMPU(keyA, 'content', err => { + let completedReplicationInfo; + async.series( + [ + next => putObjectAndCheckMD(keyA, replicationMD, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { return done(err); } - const expected = - Object.assign({}, expectedReplicationInfo, - { content: ['DATA', 'METADATA', 'MPU'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(keyA, completedReplicationInfo); return done(); - })); + }, + ); + }); - it('should update on a copy object request', done => - copyObject(keyB, keyA, true, err => { + it('should update metadata if putting a delete marker', done => + async.series( + [ + next => + putObjectAndCheckMD(keyA, newReplicationMD, err => { + if (err) { + return next(err); + } + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Set metadata to a status after replication has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + return next(); + }), + next => objectDelete(authInfo, deleteReq, log, next), + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, - expectedReplicationInfo); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + assert.strictEqual(objectMD.isDeleteMarker, true); + checkObjectReplicationInfo(keyA, replicateMetadataOnly); return done(); - })); - - it('should update on a put object ACL request', done => { - let completedReplicationInfo; - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => { - const objectMD = metadata.keyMaps - .get(bucketName).get(keyA); - // Update metadata to a status after replication - // has occurred. - objectMD.replicationInfo.status = 'COMPLETED'; - completedReplicationInfo = JSON.parse( - JSON.stringify(objectMD.replicationInfo)); - objectPutACL(authInfo, objectACLReq, log, next); - }, - ], err => { + }, + )); + + it('should not update metadata if putting a delete marker owned by ' + 'Lifecycle service account', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectDelete(authInfoLifecycleService, deleteReq, log, next), + ], + err => { if (err) { return done(err); } - checkObjectReplicationInfo(keyA, completedReplicationInfo); + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + assert.strictEqual(objectMD.isDeleteMarker, true); + checkObjectReplicationInfo(keyA, emptyReplicationMD); return done(); - }); + }, + ), + ); + + describe('Object tagging', () => { + beforeEach(done => + async.series( + [ + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => done(err), + ), + ); + + it("should update status to 'PENDING' and content to " + "'['METADATA']'if putting tag", done => { + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); }); - it('should update on a put object tagging request', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => objectPutTagging(authInfo, taggingPutReq, log, - next), - ], err => { + it("should update status to 'PENDING' and content to " + "'['METADATA']' if deleting tag", done => + async.series( + [ + // Put a new version to update replication MD content array. + next => putObjectAndCheckMD(keyA, newReplicationMD, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }, + ), + ); + }); + + describe('Complete MPU', () => { + it( + "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if completing MPU", + done => + putMPU(keyA, 'content', err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, newReplicationMD); + return done(); + }), + ); + + it( + "should update status to 'PENDING' and content to " + + "'['METADATA']' if completing MPU with 0 bytes", + done => + putMPU(keyA, '', err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }), + ); + + it('should not update replicationInfo if key does not apply', done => + putMPU(keyB, 'content', err => { if (err) { return done(err); } - const expected = Object.assign({}, - expectedReplicationInfo, - { content: ['METADATA', 'PUT_TAGGING'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(keyB, emptyReplicationMD); return done(); })); + }); - it('should update on a delete tagging request', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, next), - next => objectDeleteTagging(authInfo, taggingDeleteReq, - log, next), - ], err => { + describe('Object copy', () => { + it( + "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if copying object", + done => + copyObject(keyB, keyA, true, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, newReplicationMD); + return done(); + }), + ); + + it( + "should update status to 'PENDING' and content to " + + "'['METADATA']' if copying object with 0 bytes", + done => + copyObject(keyB, keyA, false, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, replicateMetadataOnly); + return done(); + }), + ); + + it('should not update replicationInfo if key does not apply', done => { + const copyKey = `foo-${keyA}`; + return copyObject(keyB, copyKey, true, err => { if (err) { return done(err); } - const expected = Object.assign({}, - expectedReplicationInfo, - { content: ['METADATA', 'DELETE_TAGGING'] }); - checkObjectReplicationInfo(keyA, expected); + checkObjectReplicationInfo(copyKey, emptyReplicationMD); return done(); - })); + }); + }); + }); - it('should update when putting a delete marker', done => - async.series([ - next => putObjectAndCheckMD(keyA, - expectedReplicationInfo, err => { + ['awsbackend', 'azurebackend', 'gcpbackend', 'awsbackend,azurebackend'].forEach(backend => { + const storageTypeMap = { + awsbackend: 'aws_s3', + azurebackend: 'azure', + gcpbackend: 'gcp', + 'awsbackend,azurebackend': 'aws_s3,azure', + }; + const storageType = storageTypeMap[backend]; + const backends = backend.split(',').map(site => ({ + site, + status: 'PENDING', + dataStoreVersionId: '', + })); + describe('Object metadata replicationInfo storageType value', () => { + const expectedReplicationInfo = { + status: 'PENDING', + backends, + content: ['DATA', 'METADATA'], + destination: 'arn:aws:s3:::destination-bucket', + storageClass: backend, + role: 'arn:aws:iam::account-id:role/resource', + storageType, + dataStoreVersionId: '', + isNFS: undefined, + }; + + // Expected for a metadata-only replication operation (for + // example, putting object tags). + const expectedReplicationInfoMD = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA'], + }); + + beforeEach(() => + // We have already created the bucket, so update the + // replication configuration to include a location + // constraint for the `storageClass`. This results in a + // `storageType` of 'aws_s3', for example. + Object.assign(metadata.buckets.get(bucketName), { + _replicationConfiguration: { + role: 'arn:aws:iam::account-id:role/resource', + destination: 'arn:aws:s3:::destination-bucket', + rules: [ + { + prefix: keyA, + enabled: true, + id: 'test-id', + storageClass: backend, + }, + ], + }, + }), + ); + + it('should update on a put object request', done => + putObjectAndCheckMD(keyA, expectedReplicationInfo, done)); + + it('should update on a complete MPU object request', done => + putMPU(keyA, 'content', err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['DATA', 'METADATA', 'MPU'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + })); + + it('should update on a copy object request', done => + copyObject(keyB, keyA, true, err => { + if (err) { + return done(err); + } + checkObjectReplicationInfo(keyA, expectedReplicationInfo); + return done(); + })); + + it('should update on a put object ACL request', done => { + let completedReplicationInfo; + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => { + const objectMD = metadata.keyMaps.get(bucketName).get(keyA); + // Update metadata to a status after replication + // has occurred. + objectMD.replicationInfo.status = 'COMPLETED'; + completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo)); + objectPutACL(authInfo, objectACLReq, log, next); + }, + ], + err => { if (err) { - return next(err); + return done(err); } - // Update metadata to a status indicating that - // replication has occurred for the object. - metadata - .keyMaps - .get(bucketName) - .get(keyA) - .replicationInfo - .status = 'COMPLETED'; - return next(); - }), - next => objectDelete(authInfo, deleteReq, log, next), - ], err => { - if (err) { - return done(err); - } - // Is it, in fact, a delete marker? - assert(metadata - .keyMaps - .get(bucketName) - .get(keyA) - .isDeleteMarker); - checkObjectReplicationInfo(keyA, - expectedReplicationInfoMD); - return done(); - })); + checkObjectReplicationInfo(keyA, completedReplicationInfo); + return done(); + }, + ); + }); + + it('should update on a put object tagging request', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => objectPutTagging(authInfo, taggingPutReq, log, next), + ], + err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA', 'PUT_TAGGING'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + }, + )); + + it('should update on a delete tagging request', done => + async.series( + [ + next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next), + next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + const expected = Object.assign({}, expectedReplicationInfo, { + content: ['METADATA', 'DELETE_TAGGING'], + }); + checkObjectReplicationInfo(keyA, expected); + return done(); + }, + )); + + it('should update when putting a delete marker', done => + async.series( + [ + next => + putObjectAndCheckMD(keyA, expectedReplicationInfo, err => { + if (err) { + return next(err); + } + // Update metadata to a status indicating that + // replication has occurred for the object. + metadata.keyMaps.get(bucketName).get(keyA).replicationInfo.status = 'COMPLETED'; + return next(); + }), + next => objectDelete(authInfo, deleteReq, log, next), + ], + err => { + if (err) { + return done(err); + } + // Is it, in fact, a delete marker? + assert(metadata.keyMaps.get(bucketName).get(keyA).isDeleteMarker); + checkObjectReplicationInfo(keyA, expectedReplicationInfoMD); + return done(); + }, + )); + }); }); - }); - }); + }, + ); }); diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js index 75cd3c134f..ced3c18e48 100644 --- a/tests/unit/utils/collectResponseHeaders.js +++ b/tests/unit/utils/collectResponseHeaders.js @@ -1,6 +1,5 @@ const assert = require('assert'); -const collectResponseHeaders = - require('../../../lib/utilities/collectResponseHeaders'); +const collectResponseHeaders = require('../../../lib/utilities/collectResponseHeaders'); describe('Middleware: Collect Response Headers', () => { it('should be able to set replication status when config is set', () => { @@ -21,22 +20,19 @@ describe('Middleware: Collect Response Headers', () => { }; const headers = collectResponseHeaders(objectMD); assert.deepStrictEqual(headers['x-amz-replication-status'], 'COMPLETED'); - assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], - 'COMPLETED'); + assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], 'COMPLETED'); assert.deepStrictEqual(headers['x-amz-meta-us-east-1-version-id'], '123'); - assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], - 'COMPLETED'); + assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], 'COMPLETED'); assert.deepStrictEqual(headers['x-amz-meta-us-west-2-version-id'], undefined); }); - + [ { md: { replicationInfo: null }, test: 'when config is not set' }, { md: {}, test: 'for older objects' }, ].forEach(item => { it(`should skip replication header ${item.test}`, () => { const headers = collectResponseHeaders(item.md); - assert.deepStrictEqual(headers['x-amz-replication-status'], - undefined); + assert.deepStrictEqual(headers['x-amz-replication-status'], undefined); }); }); @@ -45,19 +41,16 @@ describe('Middleware: Collect Response Headers', () => { assert.strictEqual(headers['Accept-Ranges'], 'bytes'); }); - it('should return an undefined value when x-amz-website-redirect-location' + - ' is empty', () => { + it('should return an undefined value when x-amz-website-redirect-location' + ' is empty', () => { const objectMD = { 'x-amz-website-redirect-location': '' }; const headers = collectResponseHeaders(objectMD); - assert.strictEqual(headers['x-amz-website-redirect-location'], - undefined); + assert.strictEqual(headers['x-amz-website-redirect-location'], undefined); }); it('should return the (nonempty) value of WebsiteRedirectLocation', () => { const obj = { 'x-amz-website-redirect-location': 'google.com' }; const headers = collectResponseHeaders(obj); - assert.strictEqual(headers['x-amz-website-redirect-location'], - 'google.com'); + assert.strictEqual(headers['x-amz-website-redirect-location'], 'google.com'); }); it('should not set flag when transition not in progress', () => { From 16d14885cdf2fdf4e3f680a8741e5d9182eeb308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Thu, 28 May 2026 23:42:47 +0200 Subject: [PATCH 2/5] Bump arsenal dependency Issue: CLDSRV-906 --- package.json | 2 +- yarn.lock | 90 ++++++++++++++++++++++++++-------------------------- 2 files changed, 46 insertions(+), 46 deletions(-) diff --git a/package.json b/package.json index 999993962b..235207a6f0 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.4.4", + "arsenal": "git+https://github.com/scality/Arsenal#2c429ab35a5ac82c3dafa5a0296a49a23a9c8a4a", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", diff --git a/yarn.lock b/yarn.lock index 6a6ab403de..73feb78843 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6065,32 +6065,38 @@ arraybuffer.prototype.slice@^1.0.4: get-intrinsic "^1.2.6" is-array-buffer "^3.0.4" -"arsenal@git+https://github.com/scality/Arsenal#8.2.28": - version "8.2.28" - resolved "git+https://github.com/scality/Arsenal#7df5088715bb26a62ff1db2045e611029ff17de1" +"arsenal@git+https://github.com/scality/Arsenal#2c429ab35a5ac82c3dafa5a0296a49a23a9c8a4a": + version "8.4.4" + resolved "git+https://github.com/scality/Arsenal#2c429ab35a5ac82c3dafa5a0296a49a23a9c8a4a" dependencies: - "@azure/identity" "^4.10.2" - "@azure/storage-blob" "^12.27.0" + "@aws-sdk/client-kms" "^3.975.0" + "@aws-sdk/client-s3" "^3.975.0" + "@aws-sdk/credential-providers" "^3.975.0" + "@aws-sdk/lib-storage" "^3.975.0" + "@azure/identity" "^4.13.0" + "@azure/storage-blob" "^12.31.0" "@js-sdsl/ordered-set" "^4.4.2" - "@scality/hdclient" "^1.3.1" + "@opentelemetry/api" "^1.9.0" + "@scality/hdclient" "^1.3.2" + "@smithy/node-http-handler" "^4.3.0" + "@smithy/protocol-http" "^5.3.5" JSONStream "^1.3.5" agentkeepalive "^4.6.0" ajv "6.12.3" async "~2.6.4" - aws-sdk "^2.1691.0" backo "^1.1.0" base-x "3.0.8" base62 "^2.0.2" - debug "^4.4.1" + debug "^4.4.3" fcntl "github:scality/node-fcntl#0.3.0" httpagent scality/httpagent#1.1.0 https-proxy-agent "^7.0.6" - ioredis "^5.6.1" + ioredis "^5.8.1" ipaddr.js "^2.2.0" - joi "^17.13.3" + joi "^18.0.1" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^6.17.0" + mongodb "^6.20.0" node-forge "^1.3.1" prom-client "^15.1.3" simple-glob "^0.2.0" @@ -6104,37 +6110,32 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.2.4": - version "8.2.4" - resolved "git+https://github.com/scality/Arsenal#96ef6a3e26d7528f877300606586759f1da6d0cd" +"arsenal@git+https://github.com/scality/Arsenal#8.2.28": + version "8.2.28" + resolved "git+https://github.com/scality/Arsenal#7df5088715bb26a62ff1db2045e611029ff17de1" dependencies: - "@azure/identity" "^4.5.0" - "@azure/storage-blob" "^12.25.0" - "@eslint/plugin-kit" "^0.2.3" + "@azure/identity" "^4.10.2" + "@azure/storage-blob" "^12.27.0" "@js-sdsl/ordered-set" "^4.4.2" "@scality/hdclient" "^1.3.1" - "@types/async" "^3.2.24" - "@types/utf8" "^3.0.3" JSONStream "^1.3.5" - agentkeepalive "^4.5.0" + agentkeepalive "^4.6.0" ajv "6.12.3" async "~2.6.4" aws-sdk "^2.1691.0" backo "^1.1.0" base-x "3.0.8" base62 "^2.0.2" - bson "^6.8.0" - debug "^4.3.7" - diskusage "^1.2.0" + debug "^4.4.1" fcntl "github:scality/node-fcntl#0.3.0" httpagent scality/httpagent#1.1.0 - https-proxy-agent "^7.0.5" - ioredis "^5.4.1" + https-proxy-agent "^7.0.6" + ioredis "^5.6.1" ipaddr.js "^2.2.0" joi "^17.13.3" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^6.11.0" + mongodb "^6.17.0" node-forge "^1.3.1" prom-client "^15.1.3" simple-glob "^0.2.0" @@ -6148,38 +6149,37 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.4.4": - version "8.4.4" - resolved "git+https://github.com/scality/Arsenal#a1fa5b412c0f634fa87fc7ccd7b086973d2f8d87" +"arsenal@git+https://github.com/scality/Arsenal#8.2.4": + version "8.2.4" + resolved "git+https://github.com/scality/Arsenal#96ef6a3e26d7528f877300606586759f1da6d0cd" dependencies: - "@aws-sdk/client-kms" "^3.975.0" - "@aws-sdk/client-s3" "^3.975.0" - "@aws-sdk/credential-providers" "^3.975.0" - "@aws-sdk/lib-storage" "^3.975.0" - "@azure/identity" "^4.13.0" - "@azure/storage-blob" "^12.31.0" + "@azure/identity" "^4.5.0" + "@azure/storage-blob" "^12.25.0" + "@eslint/plugin-kit" "^0.2.3" "@js-sdsl/ordered-set" "^4.4.2" - "@opentelemetry/api" "^1.9.0" - "@scality/hdclient" "^1.3.2" - "@smithy/node-http-handler" "^4.3.0" - "@smithy/protocol-http" "^5.3.5" + "@scality/hdclient" "^1.3.1" + "@types/async" "^3.2.24" + "@types/utf8" "^3.0.3" JSONStream "^1.3.5" - agentkeepalive "^4.6.0" + agentkeepalive "^4.5.0" ajv "6.12.3" async "~2.6.4" + aws-sdk "^2.1691.0" backo "^1.1.0" base-x "3.0.8" base62 "^2.0.2" - debug "^4.4.3" + bson "^6.8.0" + debug "^4.3.7" + diskusage "^1.2.0" fcntl "github:scality/node-fcntl#0.3.0" httpagent scality/httpagent#1.1.0 - https-proxy-agent "^7.0.6" - ioredis "^5.8.1" + https-proxy-agent "^7.0.5" + ioredis "^5.4.1" ipaddr.js "^2.2.0" - joi "^18.0.1" + joi "^17.13.3" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^6.20.0" + mongodb "^6.11.0" node-forge "^1.3.1" prom-client "^15.1.3" simple-glob "^0.2.0" From a7be3d6320fea77906160f0a37219a55acd853f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 01:39:46 +0200 Subject: [PATCH 3/5] Use isReplica field for x-amz-replication-status response header Prefer ReplicationInfo.isReplica over replicationInfo.status when producing x-amz-replication-status. The legacy status === 'REPLICA' branch is kept as a fallback so objects written before the feature still surface the correct value. Issue: CLDSRV-906 --- lib/utilities/collectResponseHeaders.js | 7 +++++-- tests/unit/utils/collectResponseHeaders.js | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index a8136cfe25..8480cf8d9f 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -89,8 +89,11 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagC if (objectMD.legalHold !== undefined) { responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF'; } - if (objectMD.replicationInfo && objectMD.replicationInfo.status) { - responseMetaHeaders['x-amz-replication-status'] = objectMD.replicationInfo.status; + if (objectMD.replicationInfo) { + const { isReplica, status } = objectMD.replicationInfo; + if (isReplica || status) { + responseMetaHeaders['x-amz-replication-status'] = isReplica ? 'REPLICA' : status; + } } if (Array.isArray(objectMD?.replicationInfo?.backends)) { objectMD.replicationInfo.backends.forEach(backend => { diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js index ced3c18e48..de17b64321 100644 --- a/tests/unit/utils/collectResponseHeaders.js +++ b/tests/unit/utils/collectResponseHeaders.js @@ -8,6 +8,22 @@ describe('Middleware: Collect Response Headers', () => { assert.deepStrictEqual(headers['x-amz-replication-status'], 'REPLICA'); }); + it('should set REPLICA header from isReplica even when status is PENDING', () => { + const objectMD = { + replicationInfo: { status: 'PENDING', isReplica: true }, + }; + const headers = collectResponseHeaders(objectMD); + assert.deepStrictEqual(headers['x-amz-replication-status'], 'REPLICA'); + }); + + it('should use replicationInfo.status when isReplica is false', () => { + const objectMD = { + replicationInfo: { status: 'PENDING', isReplica: false }, + }; + const headers = collectResponseHeaders(objectMD); + assert.deepStrictEqual(headers['x-amz-replication-status'], 'PENDING'); + }); + it('should set the replication status of each site', () => { const objectMD = { replicationInfo: { From d90a0f38cdb3a9ed8f3941b3914b7676c0c4a0e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 01:39:55 +0200 Subject: [PATCH 4/5] Bump microVersionId on object metadata writes Bump microVersionId on every user write that changes object metadata other than replicationInfo, providing a unique revision identifier needed by upcoming cascaded CRR loop detection. Issue: CLDSRV-906 --- lib/api/apiUtils/object/bumpMicroVersionId.js | 27 ++++++++++++++++ lib/api/objectDeleteTagging.js | 2 ++ lib/api/objectPutLegalHold.js | 2 ++ lib/api/objectPutRetention.js | 2 ++ lib/api/objectPutTagging.js | 2 ++ lib/metadata/acl.js | 3 ++ lib/services.js | 5 ++- .../api/apiUtils/object/bumpMicroVersionId.js | 31 +++++++++++++++++++ tests/unit/api/objectReplicationMD.js | 27 ++++++++++++++++ 9 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 lib/api/apiUtils/object/bumpMicroVersionId.js create mode 100644 tests/unit/api/apiUtils/object/bumpMicroVersionId.js diff --git a/lib/api/apiUtils/object/bumpMicroVersionId.js b/lib/api/apiUtils/object/bumpMicroVersionId.js new file mode 100644 index 0000000000..1a76970726 --- /dev/null +++ b/lib/api/apiUtils/object/bumpMicroVersionId.js @@ -0,0 +1,27 @@ +const { versioning } = require('arsenal'); +const { config } = require('../../../Config'); + +/** + * Bump objectMD.microVersionId. microVersionId is a generic + * metadata-revision marker, not a CRR-specific field, but cascaded CRR + * is its only consumer today - so we gate on replicationInfo to avoid + * inflating storage on objects that wouldn't use it. The gate can be + * widened later if another consumer needs it on every object. + * Pass `force = true` to bump unconditionally. + * + * @param {object} objectMD - object MD POJO or `md.getValue()` + * @param {boolean} [force] - bump even without replicationInfo + * @return {undefined} + */ +function bumpMicroVersionId(objectMD, force) { + if (!force && !objectMD?.replicationInfo) { + return; + } + + const { instanceId, replicationGroupId } = config; + + // eslint-disable-next-line no-param-reassign + objectMD.microVersionId = versioning.VersionID.generateVersionId(instanceId, replicationGroupId); +} + +module.exports = bumpMicroVersionId; diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 45d67a0f97..88c4f7187a 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -13,6 +13,7 @@ const monitoring = require('../utilities/monitoringHandler'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const { data } = require('../data/wrapper'); const { config } = require('../Config'); const REPLICATION_ACTION = 'DELETE_TAGGING'; @@ -94,6 +95,7 @@ function objectDeleteTagging(authInfo, request, log, callback) { // eslint-disable-next-line no-param-reassign objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); } + bumpMicroVersionId(objectMD); // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectTagging:Delete'; metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index d687f77ce6..b00868db02 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -8,6 +8,7 @@ const { getVersionSpecificMetadataOptions, } = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const metadata = require('../metadata/wrapper'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); @@ -106,6 +107,7 @@ function objectPutLegalHold(authInfo, request, log, callback) { // eslint-disable-next-line no-param-reassign objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); } + bumpMicroVersionId(objectMD); // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectLegalHold:Put'; metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index b8182646e9..828ad995c5 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -10,6 +10,7 @@ const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); const { pushMetric } = require('../utapi/utilities'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { config } = require('../Config'); @@ -127,6 +128,7 @@ function objectPutRetention(authInfo, request, log, callback) { if (replicationInfo) { objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); } + bumpMicroVersionId(objectMD); objectMD.originOp = 's3:ObjectRetention:Put'; /* eslint-enable no-param-reassign */ metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index 85bb652787..0d6b674167 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -11,6 +11,7 @@ const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUt const { pushMetric } = require('../utapi/utilities'); const monitoring = require('../utilities/monitoringHandler'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); const metadata = require('../metadata/wrapper'); const { data } = require('../data/wrapper'); @@ -97,6 +98,7 @@ function objectPutTagging(authInfo, request, log, callback) { // eslint-disable-next-line no-param-reassign objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo); } + bumpMicroVersionId(objectMD); // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectTagging:Put'; metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index 961b95d79f..af035c2fc9 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -1,6 +1,7 @@ const { errors } = require('arsenal'); const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo'); +const bumpMicroVersionId = require('../api/apiUtils/object/bumpMicroVersionId'); const aclUtils = require('../utilities/aclUtils'); const constants = require('../../constants'); const metadata = require('../metadata/wrapper'); @@ -59,6 +60,8 @@ const acl = { }; } + bumpMicroVersionId(objectMD); + return metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, cb); } return cb(); diff --git a/lib/services.js b/lib/services.js index 5bc5036442..6ff8e43da6 100644 --- a/lib/services.js +++ b/lib/services.js @@ -15,6 +15,7 @@ const { data } = require('./data/wrapper'); const metadata = require('./metadata/wrapper'); const { setObjectLockInformation } = require('./api/apiUtils/object/objectLockHelpers'); const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked'); +const bumpMicroVersionId = require('./api/apiUtils/object/bumpMicroVersionId'); const { parseTagFromQuery } = s3middleware.tagging; const usersBucket = constants.usersBucket; @@ -221,9 +222,7 @@ const services = { options.replayId = uploadId; } // update microVersionId when overwriting metadata. - if (updateMicroVersionId) { - md.updateMicroVersionId(); - } + bumpMicroVersionId(md.getValue(), updateMicroVersionId); // update restore if (archive) { md.setAmzStorageClass(amzStorageClass); diff --git a/tests/unit/api/apiUtils/object/bumpMicroVersionId.js b/tests/unit/api/apiUtils/object/bumpMicroVersionId.js new file mode 100644 index 0000000000..2dc50c0515 --- /dev/null +++ b/tests/unit/api/apiUtils/object/bumpMicroVersionId.js @@ -0,0 +1,31 @@ +const assert = require('assert'); + +const bumpMicroVersionId = require('../../../../../lib/api/apiUtils/object/bumpMicroVersionId'); + +describe('bumpMicroVersionId', () => { + it('should set a fresh microVersionId when replicationInfo is present', () => { + const objectMD = { replicationInfo: {} }; + bumpMicroVersionId(objectMD); + assert(objectMD.microVersionId, 'expected microVersionId to be set'); + }); + + it('should produce a different value on each call', () => { + const objectMD = { replicationInfo: {} }; + bumpMicroVersionId(objectMD); + const first = objectMD.microVersionId; + bumpMicroVersionId(objectMD); + assert.notStrictEqual(objectMD.microVersionId, first); + }); + + it('should do nothing when replicationInfo is absent', () => { + const objectMD = {}; + bumpMicroVersionId(objectMD); + assert.strictEqual(objectMD.microVersionId, undefined); + }); + + it('should bump unconditionally when force is true', () => { + const objectMD = {}; + bumpMicroVersionId(objectMD, true); + assert(objectMD.microVersionId, 'expected microVersionId to be set when force=true'); + }); +}); diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index fa228cf3cb..ca761363e5 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -1,6 +1,7 @@ const assert = require('assert'); const async = require('async'); const crypto = require('crypto'); +const { promisify } = require('util'); const BucketInfo = require('arsenal').models.BucketInfo; @@ -750,3 +751,29 @@ describe('Replication object MD without bucket replication config', () => { }, ); }); + +describe('microVersionId is bumped on every object metadata write', () => { + const getMD = key => metadata.keyMaps.get(bucketName).get(key); + const objectPutAsync = promisify(objectPut); + const objectPutTaggingAsync = promisify(objectPutTagging); + + beforeEach(() => { + cleanup(); + createBucket(); + }); + + afterEach(() => cleanup()); + + it('should set microVersionId on objectPut', async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + assert(getMD(keyA).microVersionId, 'expected microVersionId to be set'); + }); + + it('should bump microVersionId on subsequent objectPutTagging', async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + const before = getMD(keyA).microVersionId; + await objectPutTaggingAsync(authInfo, taggingPutReq, log); + const after = getMD(keyA).microVersionId; + assert(after && after !== before, 'expected microVersionId to change after tagging'); + }); +}); From 28a3bfb41e309839f92a3d143a4ec14ec57a0d04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ma=C3=ABl=20DONNART?= Date: Fri, 29 May 2026 01:37:03 +0200 Subject: [PATCH 5/5] Clear isReplica on direct user writes overwriting a replica When a user updates tags, ACL, retention, or legal-hold on an object that arrived via replication (isReplica=true), the resulting object is no longer a replica - clear the flag so the x-amz-replication-status response header reflects the new state. Issue: CLDSRV-906 --- lib/api/objectDeleteTagging.js | 4 ++++ lib/api/objectPutLegalHold.js | 4 ++++ lib/api/objectPutRetention.js | 3 +++ lib/api/objectPutTagging.js | 4 ++++ lib/metadata/acl.js | 4 ++++ tests/unit/api/objectReplicationMD.js | 26 ++++++++++++++++++++++++++ 6 files changed, 45 insertions(+) diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 88c4f7187a..22127c9e74 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -82,6 +82,10 @@ function objectDeleteTagging(authInfo, request, log, callback) { // eslint-disable-next-line no-param-reassign objectMD.tags = {}; const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + if (objectMD.replicationInfo?.isReplica) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo.isReplica = false; + } const replicationInfo = getReplicationInfo( config, objectKey, diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index b00868db02..ff0f753bf2 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -94,6 +94,10 @@ function objectPutLegalHold(authInfo, request, log, callback) { // eslint-disable-next-line no-param-reassign objectMD.legalHold = legalHold; const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + if (objectMD.replicationInfo?.isReplica) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo.isReplica = false; + } const replicationInfo = getReplicationInfo( config, objectKey, diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index 828ad995c5..6004f9742b 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -116,6 +116,9 @@ function objectPutRetention(authInfo, request, log, callback) { objectMD.retentionMode = retentionInfo.mode; objectMD.retentionDate = retentionInfo.date; const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + if (objectMD.replicationInfo?.isReplica) { + objectMD.replicationInfo.isReplica = false; + } const replicationInfo = getReplicationInfo( config, objectKey, diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index 0d6b674167..3cf5ea21c8 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -85,6 +85,10 @@ function objectPutTagging(authInfo, request, log, callback) { // eslint-disable-next-line no-param-reassign objectMD.tags = tags; const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode); + if (objectMD.replicationInfo?.isReplica) { + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo.isReplica = false; + } const replicationInfo = getReplicationInfo( config, objectKey, diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index af035c2fc9..85e8824e86 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -47,6 +47,10 @@ const acl = { objectMD.acl = addACLParams; objectMD.originOp = 's3:ObjectAcl:Put'; + if (objectMD.replicationInfo?.isReplica) { + objectMD.replicationInfo.isReplica = false; + } + // Use storageType to determine if replication update is needed, as it is set only for // "cloud" locations. This ensures that we reset replication when CRR is used, but not // when multi-backend replication (i.e. Zenko) is used. diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js index ca761363e5..9114280591 100644 --- a/tests/unit/api/objectReplicationMD.js +++ b/tests/unit/api/objectReplicationMD.js @@ -777,3 +777,29 @@ describe('microVersionId is bumped on every object metadata write', () => { assert(after && after !== before, 'expected microVersionId to change after tagging'); }); }); + +describe('isReplica is cleared on direct user writes overwriting a replica', () => { + const getMD = key => metadata.keyMaps.get(bucketName).get(key); + const objectPutAsync = promisify(objectPut); + const objectPutTaggingAsync = promisify(objectPutTagging); + + beforeEach(() => { + cleanup(); + createBucket(); + }); + + afterEach(() => cleanup()); + + it('should clear isReplica when prior MD has it true', async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + getMD(keyA).replicationInfo.isReplica = true; + await objectPutTaggingAsync(authInfo, taggingPutReq, log); + assert.strictEqual(getMD(keyA).replicationInfo.isReplica, false); + }); + + it('should leave isReplica untouched when prior MD does not have it', async () => { + await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log); + await objectPutTaggingAsync(authInfo, taggingPutReq, log); + assert.strictEqual(getMD(keyA).replicationInfo.isReplica, undefined); + }); +});