diff --git a/lib/models/ObjectMD.ts b/lib/models/ObjectMD.ts index 39fc74f5b..1c25ab2b6 100644 --- a/lib/models/ObjectMD.ts +++ b/lib/models/ObjectMD.ts @@ -1,4 +1,3 @@ -import * as crypto from 'crypto'; import * as constants from '../constants'; import * as VersionIDUtils from '../versioning/VersionID'; import { VersioningConstants } from '../versioning/constants'; @@ -32,6 +31,7 @@ export type ReplicationInfo = { storageType: string; dataStoreVersionId: string; isNFS?: boolean; + isReplica?: boolean; }; export type ObjectMDTag = { @@ -243,6 +243,7 @@ export default class ObjectMD { storageType: '', dataStoreVersionId: '', isNFS: undefined, + isReplica: undefined, }, dataStoreName: '', originOp: '', @@ -1102,9 +1103,20 @@ export default class ObjectMD { storageType?: string; dataStoreVersionId?: string; isNFS?: boolean; + isReplica?: boolean; }) { - const { status, backends, content, destination, storageClass, role, storageType, dataStoreVersionId, isNFS } = - replicationInfo; + const { + status, + backends, + content, + destination, + storageClass, + role, + storageType, + dataStoreVersionId, + isNFS, + isReplica, + } = replicationInfo; this._data.replicationInfo = { status, backends, @@ -1115,6 +1127,7 @@ export default class ObjectMD { storageType: storageType || '', dataStoreVersionId: dataStoreVersionId || '', isNFS, + isReplica: isReplica !== undefined ? isReplica : (status === 'REPLICA' ? true : undefined), }; return this; } @@ -1130,6 +1143,9 @@ export default class ObjectMD { setReplicationStatus(status: string) { this._data.replicationInfo.status = status; + if (status === 'REPLICA') { + this.setReplicationIsReplica(true); + } return this; } @@ -1151,6 +1167,27 @@ export default class ObjectMD { return !!this._data.replicationInfo.isNFS; } + /** + * Mark this object as the result of a replication write (replica), + * as opposed to a write originating from a user request. + * + * @param isReplica - true if this object was written by replication + * @return itself + */ + setReplicationIsReplica(isReplica: boolean) { + this._data.replicationInfo.isReplica = isReplica; + return this; + } + + /** + * Get whether this object was written by replication (replica). + * @return true if this object is a replica + */ + getReplicationIsReplica(): boolean { + return this._data.replicationInfo.isReplica === true + || this._data.replicationInfo.status === 'REPLICA'; + } + setReplicationSiteStatus(site: string, status: string) { const backend = this._data.replicationInfo.backends.find(o => o.site === site); if (backend) { @@ -1327,35 +1364,28 @@ export default class ObjectMD { } /** - * Create or update the microVersionId field - * - * This field can be used to force an update in MongoDB. This can - * be needed in the following cases: - * - * - in case no other metadata field changes - * - * - to detect a change when fields change but object version does - * not change e.g. when ingesting a putObjectTagging coming from - * S3C to Zenko - * - * - to manage conflicts during concurrent updates, using - * conditions on the microVersionId field. - * - * It's a field of 16 hexadecimal characters randomly generated - * + * Update the microVersionId + * + * This field can be used to force an update in MongoDB when no other + * metadata field changes, to detect a change for CRR, + * and to manage conflicts during concurrent updates using conditions on this field. + * + * @param instanceId - instance identifier (e.g. config.instanceId) + * @param replicationGroupId - replication group ID (e.g. config.replicationGroupId) * @return itself */ - updateMicroVersionId() { - this._data.microVersionId = crypto.randomBytes(8).toString('hex'); + updateMicroVersionId(instanceId: string, replicationGroupId: string) { + this._data.microVersionId = VersionIDUtils.generateVersionId(instanceId, replicationGroupId); + return this; } /** - * Get the microVersionId field, or null if not set + * Get the microVersionId field, or undefined if not set * - * @return the microVersionId field if exists, or {null} if it does not exist + * @return the microVersionId field if set, or undefined if not */ getMicroVersionId() { - return this._data.microVersionId || null; + return this._data.microVersionId || undefined; } /** diff --git a/lib/versioning/VersionID.ts b/lib/versioning/VersionID.ts index f18e3159c..4ed67557c 100644 --- a/lib/versioning/VersionID.ts +++ b/lib/versioning/VersionID.ts @@ -369,3 +369,40 @@ export function decode(str: string): string | Error { } return new Error(`cannot decode str ${str.length}`); } + +/** + * Returns true if a stored (raw) microVersionId is time-ordered and comparable. + * + * Returns false for: + * - null / undefined (field absent) + * - the legacy 16-char random-hex value written by old ObjectMD code + * - any other string shorter than the minimum valid raw length (27 chars) + */ +export function isMicroVersionIdComparable(id: string | null | undefined): boolean { + return typeof id === 'string' && id.length >= LEGACY_BASE62_DECODED_LENGTH; +} + +/** + * Compare an incoming (raw) microVersionId against the one stored in object + * metadata for CRR cascade loop/stale detection. + * + * Returns 'proceed' whenever either ID is non-comparable (absent, legacy + * 16-char hex, or otherwise too short) : a legacy incoming ID must never + * produce a false 'stale' or 'loop' event + */ +export function checkCrrCascadeEvent( + incomingRaw: string, + existingRaw: string | null | undefined, +): 'loop' | 'stale' | 'proceed' { + if (!isMicroVersionIdComparable(incomingRaw) || !isMicroVersionIdComparable(existingRaw)) { + return 'proceed'; + } + const existing = existingRaw as string; + if (incomingRaw === existing) { + return 'loop'; + } + if (incomingRaw > existing) { + return 'stale'; + } + return 'proceed'; +} diff --git a/tests/unit/models/ObjectMD.spec.js b/tests/unit/models/ObjectMD.spec.js index 0e8866aeb..4f1d4020e 100644 --- a/tests/unit/models/ObjectMD.spec.js +++ b/tests/unit/models/ObjectMD.spec.js @@ -2,8 +2,7 @@ const assert = require('assert'); const ObjectMD = require('../../../lib/models/ObjectMD').default; const ObjectMDChecksum = require('../../../lib/models/ObjectMDChecksum').default; const constants = require('../../../lib/constants'); -const ExternalNullVersionId = require('../../../lib/versioning/constants') - .VersioningConstants.ExternalNullVersionId; +const ExternalNullVersionId = require('../../../lib/versioning/constants').VersioningConstants.ExternalNullVersionId; const retainDate = new Date(); retainDate.setDate(retainDate.getDate() + 1); @@ -53,20 +52,27 @@ describe('ObjectMD class setters/getters', () => { ['AmzEncryptionKeyId', 'encryption-key-id'], ['AmzEncryptionCustomerAlgorithm', null, ''], ['AmzEncryptionCustomerAlgorithm', 'customer-algorithm'], - ['Acl', null, { - Canned: 'private', - FULL_CONTROL: [], - WRITE_ACP: [], - READ: [], - READ_ACP: [], - }], - ['Acl', { - Canned: 'public', - FULL_CONTROL: ['id'], - WRITE_ACP: ['id'], - READ: ['id'], - READ_ACP: ['id'], - }], + [ + 'Acl', + null, + { + Canned: 'private', + FULL_CONTROL: [], + WRITE_ACP: [], + READ: [], + READ_ACP: [], + }, + ], + [ + 'Acl', + { + Canned: 'public', + FULL_CONTROL: ['id'], + WRITE_ACP: ['id'], + READ: ['id'], + READ_ACP: ['id'], + }, + ], ['Key', null, ''], ['Key', 'key'], ['Location', null, []], @@ -84,55 +90,73 @@ describe('ObjectMD class setters/getters', () => { ['VersionId', null, undefined], ['VersionId', '111111'], ['Tags', null, {}], - ['Tags', { - key: 'value', - }], + [ + 'Tags', + { + key: 'value', + }, + ], ['Tags', null, {}], ['UploadId', null, undefined], ['UploadId', 'abcdefghi'], - ['ReplicationInfo', null, { - status: '', - backends: [], - content: [], - destination: '', - storageClass: '', - role: '', - storageType: '', - dataStoreVersionId: '', - isNFS: undefined, - }], - ['ReplicationInfo', { - status: 'PENDING', - backends: [{ - site: 'zenko', + [ + 'ReplicationInfo', + null, + { + status: '', + backends: [], + content: [], + destination: '', + storageClass: '', + role: '', + storageType: '', + dataStoreVersionId: '', + isNFS: undefined, + isReplica: undefined, + }, + ], + [ + 'ReplicationInfo', + { status: 'PENDING', - dataStoreVersionId: 'a', - }], - content: ['DATA', 'METADATA'], - destination: 'destination-bucket', - storageClass: 'STANDARD', - role: 'arn:aws:iam::account-id:role/src-resource,' + - 'arn:aws:iam::account-id:role/dest-resource', - storageType: 'aws_s3', - dataStoreVersionId: '', - isNFS: undefined, - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: 'a', + }, + ], + content: ['DATA', 'METADATA'], + destination: 'destination-bucket', + storageClass: 'STANDARD', + role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource', + storageType: 'aws_s3', + dataStoreVersionId: '', + isNFS: undefined, + isReplica: undefined, + }, + ], ['DataStoreName', null, ''], ['ReplicationIsNFS', null, false], ['ReplicationIsNFS', true], - ['AzureInfo', { - containerPublicAccess: 'container', - containerStoredAccessPolicies: [], - containerImmutabilityPolicy: {}, - containerLegalHoldStatus: false, - containerDeletionInProgress: false, - blobType: 'BlockBlob', - blobContentMD5: 'ABCDEF==', - blobCopyInfo: {}, - blobSequenceNumber: 42, - blobAccessTierChangeTime: 'abcdef', - blobUncommitted: false, - }], + ['ReplicationIsReplica', null, false], + ['ReplicationIsReplica', true], + [ + 'AzureInfo', + { + containerPublicAccess: 'container', + containerStoredAccessPolicies: [], + containerImmutabilityPolicy: {}, + containerLegalHoldStatus: false, + containerDeletionInProgress: false, + blobType: 'BlockBlob', + blobContentMD5: 'ABCDEF==', + blobCopyInfo: {}, + blobSequenceNumber: 42, + blobAccessTierChangeTime: 'abcdef', + blobUncommitted: false, + }, + ], ['LegalHold', null, false], ['LegalHold', true], ['RetentionMode', 'GOVERNANCE'], @@ -153,8 +177,7 @@ describe('ObjectMD class setters/getters', () => { md[`set${property}`](testValue); } const value = md[`get${property}`](); - if ((testValue !== null && typeof testValue === 'object') || - typeof defaultValue === 'object') { + if ((testValue !== null && typeof testValue === 'object') || typeof defaultValue === 'object') { assert.deepStrictEqual(value, testValue || defaultValue); } else if (testValue !== null) { assert.strictEqual(value, testValue); @@ -164,33 +187,72 @@ describe('ObjectMD class setters/getters', () => { }); }); + it('getReplicationIsReplica: returns true when status is REPLICA (backward compat)', () => { + // Old objects have no isReplica field but status === 'REPLICA'. + // getReplicationIsReplica must still return true for them. + md.setReplicationStatus('REPLICA'); + assert.strictEqual(md.getReplicationIsReplica(), true, + 'status REPLICA alone must be enough'); + // Also verify the flag is set + assert.strictEqual(md.getReplicationInfo().isReplica, true, + 'setReplicationStatus REPLICA should set the isReplica flag'); + }); + + it('getReplicationIsReplica: survives status transition to PENDING (cascade)', () => { + // After cascade fires the status becomes PENDING, but isReplica must stay true. + md.setReplicationStatus('REPLICA'); + md.setReplicationStatus('PENDING'); + assert.strictEqual(md.getReplicationIsReplica(), true, + 'isReplica must survive transition from REPLICA to PENDING'); + }); + + it('setReplicationInfo: auto-sets isReplica when status is REPLICA', () => { + md.setReplicationInfo({ + status: 'REPLICA', + backends: [], + content: [], + destination: '', + role: '', + }); + assert.strictEqual(md.getReplicationInfo().isReplica, true, + 'setReplicationInfo with status REPLICA should auto-set isReplica'); + }); + it('ObjectMD::setReplicationSiteStatus', () => { md.setReplicationInfo({ - backends: [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: 'a', - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: 'a', + }, + ], }); md.setReplicationSiteStatus('zenko', 'COMPLETED'); - assert.deepStrictEqual(md.getReplicationInfo().backends, [{ - site: 'zenko', - status: 'COMPLETED', - dataStoreVersionId: 'a', - }]); + assert.deepStrictEqual(md.getReplicationInfo().backends, [ + { + site: 'zenko', + status: 'COMPLETED', + dataStoreVersionId: 'a', + }, + ]); }); it('ObjectMD::setReplicationBackends', () => { - md.setReplicationBackends([{ - site: 'a', - status: 'b', - dataStoreVersionId: 'c', - }]); - assert.deepStrictEqual(md.getReplicationBackends(), [{ - site: 'a', - status: 'b', - dataStoreVersionId: 'c', - }]); + md.setReplicationBackends([ + { + site: 'a', + status: 'b', + dataStoreVersionId: 'c', + }, + ]); + assert.deepStrictEqual(md.getReplicationBackends(), [ + { + site: 'a', + status: 'b', + dataStoreVersionId: 'c', + }, + ]); }); it('ObjectMD::setReplicationStorageType', () => { @@ -217,41 +279,48 @@ describe('ObjectMD class setters/getters', () => { it('ObjectMD::getReplicationSiteStatus', () => { md.setReplicationInfo({ - backends: [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: 'a', - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: 'a', + }, + ], }); assert.strictEqual(md.getReplicationSiteStatus('zenko'), 'PENDING'); }); it('ObjectMD::setReplicationSiteDataStoreVersionId', () => { md.setReplicationInfo({ - backends: [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: 'a', - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: 'a', + }, + ], }); md.setReplicationSiteDataStoreVersionId('zenko', 'b'); - assert.deepStrictEqual(md.getReplicationInfo().backends, [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: 'b', - }]); + assert.deepStrictEqual(md.getReplicationInfo().backends, [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: 'b', + }, + ]); }); it('ObjectMD::getReplicationSiteDataStoreVersionId', () => { md.setReplicationInfo({ - backends: [{ - site: 'zenko', - status: 'PENDING', - dataStoreVersionId: 'a', - }], + backends: [ + { + site: 'zenko', + status: 'PENDING', + dataStoreVersionId: 'a', + }, + ], }); - assert.strictEqual( - md.getReplicationSiteDataStoreVersionId('zenko'), 'a'); + assert.strictEqual(md.getReplicationSiteDataStoreVersionId('zenko'), 'a'); }); it('ObjectMd::isMultipartUpload', () => { @@ -272,7 +341,7 @@ describe('ObjectMD class setters/getters', () => { // This one should be changed to 'x-amz-meta-foobar' 'x-ms-meta-foobar': 'bar', // ACLs are updated - 'acl': { + acl: { FULL_CONTROL: ['john'], }, }); @@ -295,22 +364,22 @@ describe('ObjectMD class setters/getters', () => { }); it('ObjectMD::microVersionId unset', () => { - assert.strictEqual(md.getMicroVersionId(), null); + assert.strictEqual(md.getMicroVersionId(), undefined); }); it('ObjectMD::microVersionId set', () => { - const generatedIds = new Set(); + const generatedIds = []; for (let i = 0; i < 100; ++i) { - md.updateMicroVersionId(); - generatedIds.add(md.getMicroVersionId()); + md.updateMicroVersionId('instance', 'RG001'); + generatedIds.push(md.getMicroVersionId()); } // all generated IDs should be different - assert.strictEqual(generatedIds.size, 100); - generatedIds.forEach(key => { - // length is always 16 in hex because leading 0s are - // also encoded in the 8-byte random buffer. - assert.strictEqual(key.length, 16); - }); + assert.strictEqual(new Set(generatedIds).size, 100); + // microVersionIds use the versionId format (reversed time ordered): + // newer values sort before older ones lexicographically. + for (let i = 1; i < generatedIds.length; ++i) { + assert(generatedIds[i] < generatedIds[i - 1]); + } }); it('ObjectMD::set/getRetentionMode', () => { @@ -420,10 +489,8 @@ describe('ObjectMD import from stored blob', () => { assert.strictEqual(importedRes.error, undefined); const importedMd = importedRes.result; const valueImported = importedMd.getValue(); - assert.strictEqual(valueImported['md-model-version'], - constants.mdModelVersion); - assert.deepStrictEqual(valueImported.location, - [{ key: 'stringLocation' }]); + assert.strictEqual(valueImported['md-model-version'], constants.mdModelVersion); + assert.deepStrictEqual(valueImported.location, [{ key: 'stringLocation' }]); }); it('should keep null location as is', () => { @@ -450,21 +517,19 @@ describe('ObjectMD import from stored blob', () => { assert.strictEqual(importedRes.error, undefined); const importedMd = importedRes.result; const valueImported = importedMd.getValue(); - assert.strictEqual(valueImported['md-model-version'], - constants.mdModelVersion); + assert.strictEqual(valueImported['md-model-version'], constants.mdModelVersion); assert.notStrictEqual(valueImported.dataStoreName, undefined); }); - it('should return undefined for dataStoreVersionId if no object location', - () => { - const md = new ObjectMD(); - const value = md.getValue(); - const jsonMd = JSON.stringify(value); - const importedRes = ObjectMD.createFromBlob(jsonMd); - assert.strictEqual(importedRes.error, undefined); - const importedMd = importedRes.result; - assert.strictEqual(importedMd.getDataStoreVersionId(), undefined); - }); + it('should return undefined for dataStoreVersionId if no object location', () => { + const md = new ObjectMD(); + const value = md.getValue(); + const jsonMd = JSON.stringify(value); + const importedRes = ObjectMD.createFromBlob(jsonMd); + assert.strictEqual(importedRes.error, undefined); + const importedMd = importedRes.result; + assert.strictEqual(importedMd.getDataStoreVersionId(), undefined); + }); it('should get dataStoreVersionId if saved in object location', () => { const md = new ObjectMD(); @@ -477,8 +542,7 @@ describe('ObjectMD import from stored blob', () => { const importedRes = ObjectMD.createFromBlob(jsonMd); assert.strictEqual(importedRes.error, undefined); const importedMd = importedRes.result; - assert.strictEqual(importedMd.getDataStoreVersionId(), - dummyLocation.dataStoreVersionId); + assert.strictEqual(importedMd.getDataStoreVersionId(), dummyLocation.dataStoreVersionId); }); it('should return an error if blob is malformed JSON', () => { @@ -497,7 +561,7 @@ describe('getAttributes static method', () => { 'cache-control': true, 'content-disposition': true, 'content-encoding': true, - 'expires': true, + expires: true, 'content-length': true, 'content-type': true, 'content-md5': true, @@ -511,25 +575,25 @@ describe('getAttributes static method', () => { 'x-amz-server-side-encryption-customer-algorithm': true, 'x-amz-website-redirect-location': true, 'x-amz-scal-transition-in-progress': true, - 'acl': true, - 'key': true, - 'location': true, - 'azureInfo': true, - 'isNull': true, - 'isNull2': true, - 'nullVersionId': true, - 'nullUploadId': true, - 'isDeleteMarker': true, - 'versionId': true, - 'tags': true, - 'uploadId': true, - 'replicationInfo': true, - 'dataStoreName': true, + acl: true, + key: true, + location: true, + azureInfo: true, + isNull: true, + isNull2: true, + nullVersionId: true, + nullUploadId: true, + isDeleteMarker: true, + versionId: true, + tags: true, + uploadId: true, + replicationInfo: true, + dataStoreName: true, 'last-modified': true, 'md-model-version': true, - 'originOp': true, - 'deleted': true, - 'isPHD': true, + originOp: true, + deleted: true, + isPHD: true, }; assert.deepStrictEqual(attributes, expectedResult); }); @@ -927,5 +991,4 @@ describe('ObjectMD checksum', () => { assert.strictEqual(c.checksumValue, `${sha256Digest}-3`); assert.strictEqual(c.checksumType, 'COMPOSITE'); }); - }); diff --git a/tests/unit/versioning/VersionID.spec.js b/tests/unit/versioning/VersionID.spec.js index e41b3b9e3..bb1ea34c5 100644 --- a/tests/unit/versioning/VersionID.spec.js +++ b/tests/unit/versioning/VersionID.spec.js @@ -1,3 +1,4 @@ +const crypto = require('crypto'); const VID = require('../../../lib/versioning/VersionID'); const VersioningConstants = require('../../../lib/versioning/constants').VersioningConstants; const assert = require('assert'); @@ -188,3 +189,69 @@ describe('test generating versionIds', () => { }); }); }); + +describe('isMicroVersionIdComparable', () => { + const { isMicroVersionIdComparable, generateVersionId } = VID; + + it('should return false for null', () => { + assert.strictEqual(isMicroVersionIdComparable(null), false); + }); + + it('should return false for undefined', () => { + assert.strictEqual(isMicroVersionIdComparable(undefined), false); + }); + + it('should return false for a legacy 16-char hex microVersionId', () => { + const legacyHex = crypto.randomBytes(8).toString('hex'); + assert.strictEqual(legacyHex.length, 16); + assert.strictEqual(isMicroVersionIdComparable(legacyHex), false); + }); + + it('should return false for a string shorter than 27 chars', () => { + assert.strictEqual(isMicroVersionIdComparable('tooshort'), false); + }); + + it('should return true for a 27-char base62 versionId (legacy format)', () => { + const raw = generateVersionId('test', 'RG001'); + assert.ok(raw.length >= 27); + assert.strictEqual(isMicroVersionIdComparable(raw), true); + }); +}); + +describe('checkCrrCascadeEvent', () => { + const { checkCrrCascadeEvent, generateVersionId } = VID; + + it('should return proceed when existingRaw is absent', () => { + assert.strictEqual(checkCrrCascadeEvent(generateVersionId('test', 'RG001'), null), 'proceed'); + assert.strictEqual(checkCrrCascadeEvent(generateVersionId('test', 'RG001'), undefined), 'proceed'); + }); + + it('should return proceed when existingRaw is a legacy 16-char hex microVersionId', () => { + const legacyHex = crypto.randomBytes(8).toString('hex'); + assert.strictEqual(checkCrrCascadeEvent(generateVersionId('test', 'RG001'), legacyHex), 'proceed'); + }); + + it('should return proceed when incomingRaw is a legacy 16-char hex microVersionId', () => { + const legacyHex = crypto.randomBytes(8).toString('hex'); + assert.strictEqual(checkCrrCascadeEvent(legacyHex, generateVersionId('test', 'RG001')), 'proceed'); + }); + + it('should return loop when incomingRaw equals existingRaw', () => { + const raw = generateVersionId('test', 'RG001'); + assert.strictEqual(checkCrrCascadeEvent(raw, raw), 'loop'); + }); + + it('should return stale when incomingRaw is older than existingRaw', () => { + const older = generateVersionId('test', 'RG001'); + const newer = generateVersionId('test', 'RG001'); + assert.ok(older > newer, 'test requires two distinct versionIds in order'); + assert.strictEqual(checkCrrCascadeEvent(older, newer), 'stale'); + }); + + it('should return proceed when incomingRaw is newer than existingRaw', () => { + const older = generateVersionId('test', 'RG001'); + const newer = generateVersionId('test', 'RG001'); + assert.ok(older > newer, 'test requires two distinct versionIds in order'); + assert.strictEqual(checkCrrCascadeEvent(newer, older), 'proceed'); + }); +});