diff --git a/lib/api/apiUtils/object/versioning.js b/lib/api/apiUtils/object/versioning.js index 418224a4a3..a090f82e84 100644 --- a/lib/api/apiUtils/object/versioning.js +++ b/lib/api/apiUtils/object/versioning.js @@ -523,7 +523,6 @@ function restoreMetadata(objMD, metadataStoreParams) { * version id of the null version */ function overwritingVersioning(objMD, metadataStoreParams) { - metadataStoreParams.updateMicroVersionId = true; metadataStoreParams.amzStorageClass = objMD['x-amz-storage-class']; // set correct originOp @@ -566,6 +565,21 @@ function overwritingVersioning(objMD, metadataStoreParams) { return options; } +/** + * @param {Object} objectMD - plain object metadata (not an ObjectMD instance) +*/ +function prepareMetadataForCascadedCrr(objectMD) { + // Bump microVersionId so cascade CRR detects the change as a new revision. + // eslint-disable-next-line no-param-reassign + objectMD.microVersionId = versionIdUtils.generateVersionId( + config.instanceId, config.replicationGroupId); + if (objectMD.replicationInfo) { + // Clear isReplica, as a user modification is no longer purely a replica + // eslint-disable-next-line no-param-reassign + objectMD.replicationInfo.isReplica = false; + } +} + module.exports = { decodeVersionId, getVersionIdResHeader, @@ -577,4 +591,5 @@ module.exports = { preprocessingVersioningDelete, overwritingVersioning, decodeVID, + prepareMetadataForCascadedCrr, }; diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js index 71115ffe5a..6c42b8573e 100644 --- a/lib/api/objectDeleteTagging.js +++ b/lib/api/objectDeleteTagging.js @@ -1,7 +1,8 @@ const async = require('async'); const { errors } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } +const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions, + prepareMetadataForCascadedCrr } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -85,6 +86,7 @@ function objectDeleteTagging(authInfo, request, log, callback) { } // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectTagging:Delete'; + prepareMetadataForCascadedCrr(objectMD); metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => next(err, bucket, objectMD)); diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js index c16f2c84e8..fd9b40fff8 100644 --- a/lib/api/objectPutLegalHold.js +++ b/lib/api/objectPutLegalHold.js @@ -2,7 +2,8 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); const collectCorsHeaders = require('../utilities/collectCorsHeaders'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = +const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions, + prepareMetadataForCascadedCrr } = require('./apiUtils/object/versioning'); const getReplicationInfo = require('./apiUtils/object/getReplicationInfo'); const metadata = require('../metadata/wrapper'); @@ -96,6 +97,7 @@ function objectPutLegalHold(authInfo, request, log, callback) { } // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectLegalHold:Put'; + prepareMetadataForCascadedCrr(objectMD); metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => next(err, bucket, objectMD)); }, diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js index 6a7a2c8441..b5a1e6fa92 100644 --- a/lib/api/objectPutRetention.js +++ b/lib/api/objectPutRetention.js @@ -1,7 +1,8 @@ const async = require('async'); const { errors, errorInstances, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = +const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions, + prepareMetadataForCascadedCrr } = require('./apiUtils/object/versioning'); const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object/objectLockHelpers'); @@ -119,6 +120,7 @@ function objectPutRetention(authInfo, request, log, callback) { objectMD.replicationInfo, replicationInfo); } objectMD.originOp = 's3:ObjectRetention:Put'; + prepareMetadataForCascadedCrr(objectMD); /* eslint-enable no-param-reassign */ metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => next(err, bucket, objectMD)); diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js index ef23dcf64d..b5d25a4d20 100644 --- a/lib/api/objectPutTagging.js +++ b/lib/api/objectPutTagging.js @@ -1,7 +1,8 @@ const async = require('async'); const { errors, s3middleware } = require('arsenal'); -const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } = +const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions, + prepareMetadataForCascadedCrr } = require('./apiUtils/object/versioning'); const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils'); @@ -89,6 +90,7 @@ function objectPutTagging(authInfo, request, log, callback) { } // eslint-disable-next-line no-param-reassign objectMD.originOp = 's3:ObjectTagging:Put'; + prepareMetadataForCascadedCrr(objectMD); metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err => next(err, bucket, objectMD)); diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js index f48ab7aa42..41e47b56f2 100644 --- a/lib/metadata/acl.js +++ b/lib/metadata/acl.js @@ -1,6 +1,7 @@ const { errors } = require('arsenal'); const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo'); +const { prepareMetadataForCascadedCrr } = require('../api/apiUtils/object/versioning'); const aclUtils = require('../utilities/aclUtils'); const constants = require('../../constants'); const metadata = require('../metadata/wrapper'); @@ -56,7 +57,7 @@ const acl = { ...replicationInfo, }; } - + prepareMetadataForCascadedCrr(objectMD); return metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, cb); } return cb(); diff --git a/lib/routes/routeBackbeat.js b/lib/routes/routeBackbeat.js index 476d3e755d..1ed33acc5f 100644 --- a/lib/routes/routeBackbeat.js +++ b/lib/routes/routeBackbeat.js @@ -7,7 +7,7 @@ const joi = require('@hapi/joi'); const backbeatProxy = httpProxy.createProxyServer({ ignorePath: true, }); -const { auth, errors, errorInstances, s3middleware, s3routes, models, storage } = +const { auth, errors, errorInstances, s3middleware, s3routes, models, storage, versioning } = require('arsenal'); const { responseJSONBody } = s3routes.routesUtils; @@ -25,6 +25,7 @@ const { dataStore } = require('../api/apiUtils/object/storeObject'); const prepareRequestContexts = require( '../api/apiUtils/authorization/prepareRequestContexts'); const { decodeVersionId } = require('../api/apiUtils/object/versioning'); +const { decode, encode, checkCrrCascadeEvent } = versioning.VersionID; const locationKeysHaveChanged = require('../api/apiUtils/object/locationKeysHaveChanged'); const { standardMetadataValidateBucketAndObj, @@ -47,6 +48,7 @@ const quotaUtils = require('../api/apiUtils/quotas/quotaUtils'); const { handleAuthorizationResults } = require('../api/api'); const { versioningPreprocessing } = require('../api/apiUtils/object/versioning'); +const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo'); const {promisify} = require('util'); const versioningPreprocessingPromised = promisify(versioningPreprocessing); @@ -430,6 +432,27 @@ function putData(request, response, bucketInfo, objMd, log, callback) { log.error(errMessage); return callback(errorInstances.BadRequest.customizeDescription(errMessage)); } + + const incomingVersionIdEncoded = request.headers['x-scal-source-version-id']; + const decoded = incomingVersionIdEncoded ? decode(incomingVersionIdEncoded) : null; + const incomingVersionIdDecoded = decoded instanceof Error ? null : decoded; + if (incomingVersionIdDecoded && objMd && objMd.versionId === incomingVersionIdDecoded) { + // Skip the write if data is already at destination for this version id + // x-scal-micro-version-id header is used by Backbeat : + // non-empty : microVersionId available, use it for crr cascade detection + // empty : old object without microVersionId, can still proceed with metadata-only + log.debug('crr cascade putData: version already at destination', { + method: 'putData', + bucketName: request.bucketName, + objectKey: request.objectKey, + hasMicroVersionId: !!objMd.microVersionId, + }); + return _respondWithHeaders(response, null, + { 'x-scal-micro-version-id': objMd.microVersionId + ? encode(objMd.microVersionId) : '' }, + log, callback); + } + const context = { bucketName: request.bucketName, owner: canonicalID, @@ -541,6 +564,28 @@ function getCanonicalIdsByAccountId(accountId, log, cb) { } function putMetadata(request, response, bucketInfo, objMd, log, callback) { + const { bucketName, objectKey } = request; + + const encodedMicroVersionId = request.headers['x-scal-micro-version-id']; + const decoded = encodedMicroVersionId ? decode(encodedMicroVersionId) : null; + const incomingRaw = decoded instanceof Error ? null : decoded; + if (incomingRaw) { + const event = checkCrrCascadeEvent(incomingRaw, objMd && objMd.microVersionId); + if (event === 'loop') { + log.debug('crr cascade putMetadata: loop detected, skipping write', { + method: 'putMetadata', bucketName, objectKey, + }); + return _respondWithHeaders(response, {}, + { 'x-scal-micro-version-id-exists': 'true' }, log, callback); + } + if (event === 'stale') { + log.debug('crr cascade putMetadata: stale event, rejecting', { + method: 'putMetadata', bucketName, objectKey, + }); + return callback(errors.OperationAborted); + } + } + return _getRequestPayload(request, (err, payload) => { if (err) { return callback(err); @@ -554,14 +599,15 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { return callback(errors.MalformedPOSTRequest); } - const { headers, bucketName, objectKey } = request; + const { headers } = request; // Destination-side delete-marker replication. // We need the REPLICA status to distinguish from // source-side replication status updates that also carry isDeleteMarker=true. if (omVal.isDeleteMarker && omVal.replicationInfo - && omVal.replicationInfo.status === 'REPLICA' + && (omVal.replicationInfo.isReplica === true + || omVal.replicationInfo.status === 'REPLICA') && request.serverAccessLog) { // eslint-disable-next-line no-param-reassign request.serverAccessLog.replication = true; @@ -575,7 +621,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // URI shape. // The REPLICA status excludes source-side replication-status updates. if (omVal.replicationInfo - && omVal.replicationInfo.status === 'REPLICA' + && (omVal.replicationInfo.isReplica === true + || omVal.replicationInfo.status === 'REPLICA') && (omVal.originOp === 's3:ObjectTagging:Put' || omVal.originOp === 's3:ObjectTagging:Delete') && request.serverAccessLog) { @@ -591,7 +638,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // populates the aclRequired field. // The REPLICA status excludes source-side replication-status updates. if (omVal.replicationInfo - && omVal.replicationInfo.status === 'REPLICA' + && (omVal.replicationInfo.isReplica === true + || omVal.replicationInfo.status === 'REPLICA') && omVal.originOp === 's3:ObjectAcl:Put' && request.serverAccessLog) { // eslint-disable-next-line no-param-reassign @@ -669,7 +717,8 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { // then we want to create a version for the replica object even though // none was provided in the object metadata value. if (omVal.replicationInfo.isNFS) { - const isReplica = omVal.replicationInfo.status === 'REPLICA'; + const isReplica = omVal.replicationInfo.isReplica === true + || omVal.replicationInfo.status === 'REPLICA'; versioning = isReplica; omVal.replicationInfo.isNFS = !isReplica; } @@ -721,6 +770,53 @@ function putMetadata(request, response, bucketInfo, objMd, log, callback) { options.isNull = isNull; } + // Cascade triggering + // If the bucket receiving this replica has its own CRR rules, set + // status to PENDING so the queue populator here picks it up for the + // next hop. If not, clear the source-side replicationInfo fields + // Always mark isReplica=true. + if (incomingRaw) { + const isMDOnly = headers['x-scal-replication-content'] === 'METADATA'; + const objSize = omVal['content-length'] || 0; + + // These S3-compatible Scality locations are excluded + // as cascade targets because they use the MultiBackend S3 path which + // bypasses the putData/putMetadata routes, so loop detection cannot fire + // on those destinations. + const BLOCKED_LOCATION_TYPES = [ + 'location-scality-ring-s3-v1', + 'location-scality-artesca-s3-v1', + ]; + + const nextReplInfo = getReplicationInfo( + config, objectKey, bucketInfo, isMDOnly, objSize, + null, null, null); + + if (nextReplInfo) { + nextReplInfo.backends = nextReplInfo.backends.filter(b => { + const loc = config.locationConstraints[b.site]; + return !loc || !BLOCKED_LOCATION_TYPES.includes(loc.type); + }); + } + + if (nextReplInfo && nextReplInfo.backends.length > 0) { + omVal.replicationInfo = nextReplInfo; + } else { + omVal.replicationInfo = { + status: '', + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }; + } + + omVal.replicationInfo.isReplica = true; + } + return async.series([ // Zenko's CRR delegates replacing the account // information to the destination's Cloudserver, as diff --git a/lib/routes/utilities/pushReplicationMetric.js b/lib/routes/utilities/pushReplicationMetric.js index c81027a494..8bec307855 100644 --- a/lib/routes/utilities/pushReplicationMetric.js +++ b/lib/routes/utilities/pushReplicationMetric.js @@ -4,7 +4,7 @@ const { pushMetric } = require('../../utapi/utilities'); function getMetricToPush(prevObjectMD, newObjectMD) { // We only want to update metrics for a destination bucket. - if (newObjectMD.getReplicationStatus() !== 'REPLICA') { + if (!newObjectMD.getReplicationIsReplica()) { return null; } diff --git a/lib/services.js b/lib/services.js index 97116329a4..a8485018d9 100644 --- a/lib/services.js +++ b/lib/services.js @@ -109,9 +109,9 @@ const services = { lastModifiedDate, versioning, versionId, uploadId, tagging, taggingCopy, replicationInfo, defaultRetention, dataStoreName, creationTime, retentionMode, retentionDate, - legalHold, originOp, updateMicroVersionId, archive, oldReplayId, - deleteNullKey, amzStorageClass, overheadField, needOplogUpdate, - restoredEtag, bucketOwnerId } = params; + legalHold, originOp, archive, + oldReplayId, deleteNullKey, amzStorageClass, overheadField, + needOplogUpdate, restoredEtag, bucketOwnerId } = params; log.trace('storing object in metadata'); assert.strictEqual(typeof bucketName, 'string'); const md = new ObjectMD(); @@ -189,10 +189,7 @@ const services = { md.setUploadId(uploadId); options.replayId = uploadId; } - // update microVersionId when overwriting metadata. - if (updateMicroVersionId) { - md.updateMicroVersionId(); - } + md.updateMicroVersionId(config.instanceId, config.replicationGroupId); // update restore if (archive) { md.setAmzStorageClass(amzStorageClass); diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js index a754300c62..111ce12f2c 100644 --- a/lib/utilities/collectResponseHeaders.js +++ b/lib/utilities/collectResponseHeaders.js @@ -100,9 +100,13 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF'; } - if (objectMD.replicationInfo && objectMD.replicationInfo.status) { - responseMetaHeaders['x-amz-replication-status'] = - objectMD.replicationInfo.status; + const replInfo = objectMD.replicationInfo; + if (replInfo) { + if (replInfo.isReplica === true || replInfo.status === 'REPLICA') { + responseMetaHeaders['x-amz-replication-status'] = 'REPLICA'; + } else if (replInfo.status) { + responseMetaHeaders['x-amz-replication-status'] = replInfo.status; + } } if (Array.isArray(objectMD?.replicationInfo?.backends)) { objectMD.replicationInfo.backends.forEach(backend => { diff --git a/package.json b/package.json index e8626e7e78..4d91d51c74 100644 --- a/package.json +++ b/package.json @@ -19,21 +19,21 @@ }, "homepage": "https://github.com/scality/S3#readme", "dependencies": { + "@aws-crypto/crc32": "^5.2.0", + "@aws-crypto/crc32c": "^5.2.0", "@aws-sdk/client-iam": "^3.930.0", "@aws-sdk/client-s3": "^3.1013.0", "@aws-sdk/client-sts": "^3.930.0", + "@aws-sdk/crc64-nvme-crt": "^3.989.0", "@aws-sdk/credential-providers": "^3.864.0", "@aws-sdk/middleware-retry": "^3.374.0", "@aws-sdk/protocol-http": "^3.374.0", "@aws-sdk/s3-request-presigner": "^3.901.0", "@aws-sdk/signature-v4": "^3.374.0", - "@aws-crypto/crc32": "^5.2.0", - "@aws-crypto/crc32c": "^5.2.0", - "@aws-sdk/crc64-nvme-crt": "^3.989.0", "@azure/storage-blob": "^12.28.0", "@hapi/joi": "^17.1.1", "@smithy/node-http-handler": "^3.0.0", - "arsenal": "git+https://github.com/scality/Arsenal#8.4.2", + "arsenal": "file:../Arsenal", "async": "2.6.4", "bucketclient": "scality/bucketclient#8.2.7", "bufferutil": "^4.0.8", @@ -65,7 +65,7 @@ }, "devDependencies": { "@eslint/compat": "^1.2.2", - "@scality/cloudserverclient": "1.0.7", + "@scality/cloudserverclient": "file:../cloudserverclient", "@scality/eslint-config-scality": "scality/Guidelines#8.3.1", "eslint": "^9.14.0", "eslint-plugin-import": "^2.31.0", @@ -84,10 +84,10 @@ "nodemon": "^3.1.10", "nyc": "^15.1.0", "pino-pretty": "^13.1.3", + "prettier": "^3.4.2", "sinon": "^13.0.1", "ts-morph": "^28.0.0", - "tv4": "^1.3.0", - "prettier": "^3.4.2" + "tv4": "^1.3.0" }, "resolutions": { "jsonwebtoken": "^9.0.0", diff --git a/tests/functional/backbeat/crrCascade.js b/tests/functional/backbeat/crrCascade.js new file mode 100644 index 0000000000..fcc8852083 --- /dev/null +++ b/tests/functional/backbeat/crrCascade.js @@ -0,0 +1,306 @@ +'use strict'; + +const assert = require('assert'); +const { createHash } = require('crypto'); +const { v4: uuidv4 } = require('uuid'); +const { + CreateBucketCommand, + PutBucketReplicationCommand, + PutBucketVersioningCommand, + PutObjectCommand, +} = require('@aws-sdk/client-s3'); + +const { versioning, models } = require('arsenal'); +const { ObjectMD } = models; +const BucketUtility = require('../aws-node-sdk/lib/utility/bucket-util'); + +const { + BackbeatRoutesClient, + GetMetadataCommand, + PutMetadataCommand, + PutDataCommand, +} = require('@scality/cloudserverclient'); + +const { generateVersionId, encode: encodeVersionId } = versioning.VersionID; + +const TEST_BUCKET = `bucket-crr-cascade-${uuidv4().split('-')[0]}`; +const TEST_BUCKET_CRR = `bucket-crr-cascade-crr-${uuidv4().split('-')[0]}`; +const DEST_BUCKET = `bucket-crr-cascade-dest-${uuidv4().split('-')[0]}`; +const OBJECT_BODY = 'imAboutToBeCascadedWitNoParachuteInMyBack'; +const OBJECT_MD5_HEX = createHash('md5').update(OBJECT_BODY).digest('hex'); +const CANONICAL_ID = '79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be'; +const bucketUtil = new BucketUtility('default', {}); +const s3 = bucketUtil.s3; + +let backbeatClient; + +function makeMicroVersionId() { + const raw = generateVersionId('test-instance', 'RG001'); + return { raw, encoded: encodeVersionId(raw) }; +} + +// Build a minimal valid metadata body for putMetadata. +function buildMetadataBody(overrides) { + const obj = Object.assign({ + 'content-length': Buffer.byteLength(OBJECT_BODY), + 'content-type': 'text/plain', + 'last-modified': new Date().toISOString(), + 'x-amz-version-id': 'null', + 'owner-id': CANONICAL_ID, + 'owner-display-name': 'test', + 'content-md5': OBJECT_MD5_HEX, + replicationInfo: { + status: 'REPLICA', + isReplica: true, + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + }, + }, overrides || {}); + return Buffer.from(JSON.stringify(obj)); +} + +async function putMetadata(key, mvId) { + const bodyOverrides = mvId + ? { microVersionId: mvId.raw } + : {}; + return backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + MicroVersionId: mvId ? mvId.encoded : undefined, + Body: buildMetadataBody(bodyOverrides), + })); +} + +async function putData(key, { versionId } = {}) { + return backbeatClient.send(new PutDataCommand({ + Bucket: TEST_BUCKET, + Key: key, + ContentMD5: OBJECT_MD5_HEX, + CanonicalID: CANONICAL_ID, + VersioningRequired: true, + VersionId: versionId || undefined, + Body: Buffer.from(OBJECT_BODY), + })); +} + +before(async () => { + const creds = await s3.config.credentials(); + backbeatClient = new BackbeatRoutesClient({ + endpoint: `http://${process.env.IP || '127.0.0.1'}:8000`, + region: 'us-east-1', + credentials: { + accessKeyId: creds.accessKeyId, + secretAccessKey: creds.secretAccessKey, + }, + forcePathStyle: true, + }); + + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); + + await s3.send(new CreateBucketCommand({ Bucket: DEST_BUCKET })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: DEST_BUCKET, + VersioningConfiguration: { Status: 'Enabled' }, + })); + await s3.send(new CreateBucketCommand({ Bucket: TEST_BUCKET_CRR })); + await s3.send(new PutBucketVersioningCommand({ + Bucket: TEST_BUCKET_CRR, + VersioningConfiguration: { Status: 'Enabled' }, + })); + await s3.send(new PutBucketReplicationCommand({ + Bucket: TEST_BUCKET_CRR, + ReplicationConfiguration: { + Role: 'arn:aws:iam::account-id:role/src-resource,' + + 'arn:aws:iam::account-id:role/dest-resource', + Rules: [{ + Status: 'Enabled', + Prefix: '', + Destination: { + Bucket: `arn:aws:s3:::${DEST_BUCKET}`, + StorageClass: 'zenko', + }, + }], + }, + })); +}); + +describe('CRR cascade — putMetadata', () => { + it('should return loop-detected on second write with the same microVersionId', async () => { + const key = 'crr-cascade-md-loop'; + const mvId = makeMicroVersionId(); + const first = await putMetadata(key, mvId); + assert.ok(!first.MicroVersionIdExists, + 'should not be flagged as loop on first write'); + const second = await putMetadata(key, mvId); + assert.strictEqual(second.MicroVersionIdExists, true, + 'second write with same id should be flagged as loop'); + }); + + it('should return 409 when writing with an older microVersionId', async () => { + const key = 'crr-cascade-md-stale'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + await putMetadata(key, newerMvId); + await assert.rejects( + () => putMetadata(key, olderMvId), + err => { + assert.strictEqual(err.$metadata?.httpStatusCode, 409); + return true; + }, + ); + }); +}); + +describe('CRR cascade — putData', () => { + it('should return ExistingMicroVersionId when putData versionId matches current master', async () => { + const key = 'crr-cascade-data-version-collision'; + + // Create a versioned object so it has both a versionId and a + // microVersionId in its metadata. + const putResult = await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + const encodedVersionId = putResult.VersionId; + assert.ok(encodedVersionId, 'PutObject should return a VersionId'); + + // putData with that versionId: cloudserver detects the data is already + // present and returns the stored microVersionId instead of a Location. + const output = await putData(key, { versionId: encodedVersionId }); + assert.ok(output.ExistingMicroVersionId, + 'should return ExistingMicroVersionId when data already exists for this versionId'); + assert.ok(!output.Location, + 'should not return a Location when data write was skipped'); + + // ExistingMicroVersionId must be the actual microVersionId stored in + // metadata (decodable, same raw value). + const { Body } = await backbeatClient.send(new GetMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + })); + const storedMd = new ObjectMD(JSON.parse(Body)); + const { decode } = versioning.VersionID; + const decoded = decode(output.ExistingMicroVersionId); + assert.ok(!(decoded instanceof Error), + 'ExistingMicroVersionId should be a decodable versionId'); + assert.strictEqual(decoded, storedMd.getMicroVersionId(), + 'decoded ExistingMicroVersionId should match the stored microVersionId'); + }); + + it('should write data normally when VersionId does not match the current master', async () => { + const key = 'crr-cascade-data-version-no-match'; + + // Create an object to populate objMd so the comparison can be made. + const putResult = await s3.send(new PutObjectCommand({ + Bucket: TEST_BUCKET, + Key: key, + Body: Buffer.from(OBJECT_BODY), + ContentType: 'text/plain', + })); + assert.ok(putResult.VersionId, 'PutObject should return a VersionId'); + + // Call putData with a versionId string that differs from the master. + // Cloudserver should fall through and write the data normally. + const output = await putData(key, { versionId: 'totally-different-version-id' }); + assert.ok(!output.ExistingMicroVersionId, + 'should not return ExistingMicroVersionId when versionId does not match'); + assert.ok(output.Location, + 'should return a Location because data was written normally'); + }); +}); + +describe('CRR cascade — no errors', () => { + it('should succeed and store updated metadata when putMetadata microVersionId is newer', async () => { + const key = 'crr-cascade-md-forward'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + + await putMetadata(key, olderMvId); + const output = await putMetadata(key, newerMvId); + assert.ok(!output.MicroVersionIdExists, + 'newer write should not be flagged as loop'); + + const { Body } = await backbeatClient.send(new GetMetadataCommand({ + Bucket: TEST_BUCKET, + Key: key, + })); + const storedMd = new ObjectMD(JSON.parse(Body)); + assert.strictEqual(storedMd.getMicroVersionId(), newerMvId.raw, + 'stored microVersionId should be the newer one'); + // Cascade reset: bucket has no CRR rules so replicationInfo is cleared + // but isReplica is always preserved. + assert.strictEqual(storedMd.getReplicationStatus(), '', + 'replication status should be cleared when no further replication is configured'); + assert.deepStrictEqual(storedMd.getReplicationBackends(), [], + 'replication backends should be empty when no further replication is configured'); + assert.strictEqual(storedMd.getReplicationIsReplica(), true, + 'isReplica should be preserved regardless of further replication config'); + }); + +}); + +describe('CRR cascade — cascade to next hop', () => { + it('should set replication status to PENDING and preserve isReplica when bucket has CRR rules', async () => { + const key = 'crr-cascade-next-hop'; + const olderMvId = makeMicroVersionId(); + const newerMvId = makeMicroVersionId(); + + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: olderMvId.encoded, + Body: buildMetadataBody({ microVersionId: olderMvId.raw }), + })); + await backbeatClient.send(new PutMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + MicroVersionId: newerMvId.encoded, + Body: buildMetadataBody({ microVersionId: newerMvId.raw }), + })); + + const { Body } = await backbeatClient.send(new GetMetadataCommand({ + Bucket: TEST_BUCKET_CRR, + Key: key, + })); + const storedMd = new ObjectMD(JSON.parse(Body)); + assert.strictEqual(storedMd.getMicroVersionId(), newerMvId.raw, + 'stored microVersionId should be the newer one'); + // Cascade triggered: bucket has a matching CRR rule, so status is PENDING + // for the next hop, with the destination backend populated. + assert.strictEqual(storedMd.getReplicationStatus(), 'PENDING', + 'replication status should be PENDING when a CRR rule matches'); + assert.ok(storedMd.getReplicationBackends().length > 0, + 'replication backends should be populated when a CRR rule matches'); + assert.strictEqual(storedMd.getReplicationIsReplica(), true, + 'isReplica should be preserved regardless of further replication config'); + }); +}); + +describe('CRR cascade — baseline (no cascade headers)', () => { + it('should succeed normally when putData has no VersionId header', async () => { + const key = 'crr-cascade-baseline-data'; + const output = await putData(key); + assert.ok(!output.ExistingMicroVersionId, + 'putData without VersionId should not set ExistingMicroVersionId'); + assert.ok(output.Location, + 'putData without VersionId should return a Location'); + }); + + it('should succeed normally when putMetadata has no MicroVersionId header', async () => { + const key = 'crr-cascade-baseline-md'; + const output = await putMetadata(key, null); + assert.ok(!output.MicroVersionIdExists, + 'putMetadata without MicroVersionId should not set MicroVersionIdExists'); + }); +}); diff --git a/tests/unit/routes/routeBackbeat.js b/tests/unit/routes/routeBackbeat.js index 5d21d61a70..597d57ba20 100644 --- a/tests/unit/routes/routeBackbeat.js +++ b/tests/unit/routes/routeBackbeat.js @@ -174,6 +174,48 @@ describe('routeBackbeat', () => { assert.deepStrictEqual(mockResponse.body, [{}]); }); + it('should skip data write when x-scal-source-version-id matches master with no microVersionId', async () => { + // Old objects without microVersionId: data is already at destination. + // Cloudserver skips the write and returns null body with no header. + // Backbeat detects Location===null and treats it as partAlreadyAtDest. + const { versioning } = require('arsenal'); + const rawVersionId = versioning.VersionID.generateVersionId('test', 'RG001'); + const encodedVersionId = versioning.VersionID.encode(rawVersionId); + + mockRequest = prepareDummyRequest({ + 'x-scal-canonical-id': 'id', + 'content-md5': '1234', + 'content-length': '0', + 'x-scal-versioning-required': 'true', + 'x-scal-source-version-id': encodedVersionId, + }); + mockRequest.method = 'PUT'; + mockRequest.url = '/_/backbeat/data/bucket0/key0'; + mockRequest.destroy = () => {}; + + metadataUtils.standardMetadataValidateBucketAndObj.callsFake( + (params, denies, log, callback) => { + callback(null, { + getVersioningConfiguration: () => ({ Status: 'Enabled' }), + isVersioningEnabled: () => true, + getLocationConstraint: () => undefined, + }, { + versionId: rawVersionId, + // no microVersionId — old-format object + }); + }); + + routeBackbeat('127.0.0.1', mockRequest, mockResponse, log); + void await endPromise; + + sinon.assert.notCalled(storeObject.dataStore); + assert.strictEqual(mockResponse.statusCode, 200); + assert.strictEqual(mockResponse.body, null, 'should return null body (no Location)'); + const [, responseHeaders] = mockResponse.writeHead.firstCall.args; + assert.strictEqual(responseHeaders['x-scal-micro-version-id'], '', + 'should have empty x-scal-micro-version-id header for old-format objects'); + }); + describe('putMetadata', () => { const bucketInfo = { getVersioningConfiguration: () => ({ Status: 'Enabled' }), diff --git a/yarn.lock b/yarn.lock index 66231c1951..7f05e39a67 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3227,14 +3227,22 @@ resolved "https://registry.yarnpkg.com/@rtsao/scc/-/scc-1.1.0.tgz#927dd2fae9bc3361403ac2c7a00c32ddce9ad7e8" integrity sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g== -"@scality/cloudserverclient@1.0.7": - version "1.0.7" - resolved "https://registry.yarnpkg.com/@scality/cloudserverclient/-/cloudserverclient-1.0.7.tgz#ee9eed09cc7da5e97d5ad8359f429218a0a30859" - integrity sha512-gMDtI/ufRDVqWJlYkvqxXRfZOChBCw9uXt2lsaIvMuiqx/pDTNZMVLfPyRN5vx0tb6HP+5ZAe2FyPdtKXG24ng== +"@scality/cloudserverclient@file:../cloudserverclient": + version "1.0.9" + dependencies: + "@aws-sdk/client-s3" "^3.1009.0" + "@aws-sdk/middleware-expect-continue" "^3.972.8" + JSONStream "^1.3.5" + fast-xml-parser "^5.5.7" + +"@scality/cloudserverclient@file:../cloudserverclient/scality-cloudserverclient-v1.0.9.tgz": + version "1.0.9" + resolved "file:../cloudserverclient/scality-cloudserverclient-v1.0.9.tgz#41b761c46e0e620abbb00cc3fe2844b2e8fc84b8" dependencies: "@aws-sdk/client-s3" "^3.1009.0" + "@aws-sdk/middleware-expect-continue" "^3.972.8" JSONStream "^1.3.5" - fast-xml-parser "^4.3.2" + fast-xml-parser "^5.5.7" "@scality/eslint-config-scality@scality/Guidelines#8.3.1": version "8.3.1" @@ -6060,32 +6068,36 @@ arraybuffer.prototype.slice@^1.0.4: get-intrinsic "^1.2.6" is-array-buffer "^3.0.4" -"arsenal@git+https://github.com/scality/Arsenal#8.2.28": - version "8.2.28" - resolved "git+https://github.com/scality/Arsenal#7df5088715bb26a62ff1db2045e611029ff17de1" +"arsenal@file:../Arsenal": + version "8.4.2" dependencies: - "@azure/identity" "^4.10.2" - "@azure/storage-blob" "^12.27.0" + "@aws-sdk/client-kms" "^3.975.0" + "@aws-sdk/client-s3" "^3.975.0" + "@aws-sdk/credential-providers" "^3.975.0" + "@aws-sdk/lib-storage" "^3.975.0" + "@azure/identity" "^4.13.0" + "@azure/storage-blob" "^12.31.0" "@js-sdsl/ordered-set" "^4.4.2" - "@scality/hdclient" "^1.3.1" + "@scality/hdclient" "^1.3.2" + "@smithy/node-http-handler" "^4.3.0" + "@smithy/protocol-http" "^5.3.5" JSONStream "^1.3.5" agentkeepalive "^4.6.0" ajv "6.12.3" async "~2.6.4" - aws-sdk "^2.1691.0" backo "^1.1.0" base-x "3.0.8" base62 "^2.0.2" - debug "^4.4.1" + debug "^4.4.3" fcntl "github:scality/node-fcntl#0.3.0" httpagent scality/httpagent#1.1.0 https-proxy-agent "^7.0.6" - ioredis "^5.6.1" + ioredis "^5.8.1" ipaddr.js "^2.2.0" - joi "^17.13.3" + joi "^18.0.1" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^6.17.0" + mongodb "^6.20.0" node-forge "^1.3.1" prom-client "^15.1.3" simple-glob "^0.2.0" @@ -6099,37 +6111,32 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.2.4": - version "8.2.4" - resolved "git+https://github.com/scality/Arsenal#96ef6a3e26d7528f877300606586759f1da6d0cd" +"arsenal@git+https://github.com/scality/Arsenal#8.2.28": + version "8.2.28" + resolved "git+https://github.com/scality/Arsenal#7df5088715bb26a62ff1db2045e611029ff17de1" dependencies: - "@azure/identity" "^4.5.0" - "@azure/storage-blob" "^12.25.0" - "@eslint/plugin-kit" "^0.2.3" + "@azure/identity" "^4.10.2" + "@azure/storage-blob" "^12.27.0" "@js-sdsl/ordered-set" "^4.4.2" "@scality/hdclient" "^1.3.1" - "@types/async" "^3.2.24" - "@types/utf8" "^3.0.3" JSONStream "^1.3.5" - agentkeepalive "^4.5.0" + agentkeepalive "^4.6.0" ajv "6.12.3" async "~2.6.4" aws-sdk "^2.1691.0" backo "^1.1.0" base-x "3.0.8" base62 "^2.0.2" - bson "^6.8.0" - debug "^4.3.7" - diskusage "^1.2.0" + debug "^4.4.1" fcntl "github:scality/node-fcntl#0.3.0" httpagent scality/httpagent#1.1.0 - https-proxy-agent "^7.0.5" - ioredis "^5.4.1" + https-proxy-agent "^7.0.6" + ioredis "^5.6.1" ipaddr.js "^2.2.0" joi "^17.13.3" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^6.11.0" + mongodb "^6.17.0" node-forge "^1.3.1" prom-client "^15.1.3" simple-glob "^0.2.0" @@ -6143,37 +6150,37 @@ arraybuffer.prototype.slice@^1.0.4: optionalDependencies: ioctl "^2.0.2" -"arsenal@git+https://github.com/scality/Arsenal#8.4.2": - version "8.4.2" - resolved "git+https://github.com/scality/Arsenal#2312744c5e6bdd4d0cc9d60bf4cdcf10f32461e6" +"arsenal@git+https://github.com/scality/Arsenal#8.2.4": + version "8.2.4" + resolved "git+https://github.com/scality/Arsenal#96ef6a3e26d7528f877300606586759f1da6d0cd" dependencies: - "@aws-sdk/client-kms" "^3.975.0" - "@aws-sdk/client-s3" "^3.975.0" - "@aws-sdk/credential-providers" "^3.975.0" - "@aws-sdk/lib-storage" "^3.975.0" - "@azure/identity" "^4.13.0" - "@azure/storage-blob" "^12.31.0" + "@azure/identity" "^4.5.0" + "@azure/storage-blob" "^12.25.0" + "@eslint/plugin-kit" "^0.2.3" "@js-sdsl/ordered-set" "^4.4.2" - "@scality/hdclient" "^1.3.2" - "@smithy/node-http-handler" "^4.3.0" - "@smithy/protocol-http" "^5.3.5" + "@scality/hdclient" "^1.3.1" + "@types/async" "^3.2.24" + "@types/utf8" "^3.0.3" JSONStream "^1.3.5" - agentkeepalive "^4.6.0" + agentkeepalive "^4.5.0" ajv "6.12.3" async "~2.6.4" + aws-sdk "^2.1691.0" backo "^1.1.0" base-x "3.0.8" base62 "^2.0.2" - debug "^4.4.3" + bson "^6.8.0" + debug "^4.3.7" + diskusage "^1.2.0" fcntl "github:scality/node-fcntl#0.3.0" httpagent scality/httpagent#1.1.0 - https-proxy-agent "^7.0.6" - ioredis "^5.8.1" + https-proxy-agent "^7.0.5" + ioredis "^5.4.1" ipaddr.js "^2.2.0" - joi "^18.0.1" + joi "^17.13.3" level "~5.0.1" level-sublevel "~6.6.5" - mongodb "^6.20.0" + mongodb "^6.11.0" node-forge "^1.3.1" prom-client "^15.1.3" simple-glob "^0.2.0" @@ -7850,7 +7857,7 @@ fast-xml-builder@^1.1.4: dependencies: path-expression-matcher "^1.1.3" -fast-xml-parser@5.2.5, fast-xml-parser@5.3.6, fast-xml-parser@5.5.6, fast-xml-parser@^4.3.2, fast-xml-parser@^5.0.7, fast-xml-parser@^5.5.6: +fast-xml-parser@5.2.5, fast-xml-parser@5.3.6, fast-xml-parser@5.5.6, fast-xml-parser@^5.0.7, fast-xml-parser@^5.5.6, fast-xml-parser@^5.5.7: version "5.5.7" resolved "https://registry.yarnpkg.com/fast-xml-parser/-/fast-xml-parser-5.5.7.tgz#e1ddc86662d808450a19cf2fb6ccc9c3c9933c5d" integrity sha512-LteOsISQ2GEiDHZch6L9hB0+MLoYVLToR7xotrzU0opCICBkxOPgHAy1HxAvtxfJNXDJpgAsQN30mkrfpO2Prg==