diff --git a/extensions/replication/tasks/ReplicateObject.js b/extensions/replication/tasks/ReplicateObject.js index 34a1d2496..a67de5bcd 100644 --- a/extensions/replication/tasks/ReplicateObject.js +++ b/extensions/replication/tasks/ReplicateObject.js @@ -1,9 +1,9 @@ const async = require('async'); const { S3Client, GetBucketReplicationCommand, GetObjectCommand } = require('@aws-sdk/client-s3'); -const errors = require('arsenal').errors; -const jsutil = require('arsenal').jsutil; -const ObjectMDLocation = require('arsenal').models.ObjectMDLocation; +const { errors, jsutil, models, versioning } = require('arsenal'); +const ObjectMDLocation = models.ObjectMDLocation; +const { decode, checkCrrCascadeEvent } = versioning.VersionID; const ClientManager = require('../../../lib/clients/ClientManager'); const BackbeatMetadataProxy = require('../../../lib/BackbeatMetadataProxy'); @@ -30,6 +30,9 @@ const ObjectQueueEntry = require('../../../lib/models/ObjectQueueEntry'); const { authTypeAssumeRole } = require('../../../lib/constants'); const errorAlreadyCompleted = {}; +const cascadeLoopDetected = {}; +const cascadeDataComplete = {}; +const partAlreadyAtDest = {}; function _extractAccountIdFromRole(role) { return role.split(':')[4]; @@ -422,11 +425,16 @@ class ReplicateObject extends BackbeatTask { const mpuConcLimit = this.repConfig.queueProcessor.mpuPartsConcurrency; return mapLimitWaitPendingIfError(locations, mpuConcLimit, (part, done) => { this._getAndPutPart(sourceEntry, destEntry, part, log, done); - }, (err, destLocations) => { + }, (err, partResults) => { + const allPartsAlreadyAtDest = !err && + partResults.length > 0 && + partResults.every(result => result === partAlreadyAtDest); + const destLocations = allPartsAlreadyAtDest ? [] : + (partResults || []).filter(result => result && result !== partAlreadyAtDest); if (err) { return this._deleteOrphans(destEntry, destLocations, log, () => cb(err)); } - return cb(null, destLocations); + return cb(null, destLocations, allPartsAlreadyAtDest); }); } @@ -540,6 +548,7 @@ class ReplicateObject extends BackbeatTask { // destination bucket has to be versioning enabled. VersioningRequired: true, RequestUids: log.getSerializedUids(), + VersionId: sourceEntry.getEncodedVersionId(), }); addContentLengthMiddleware( putCommand, @@ -548,6 +557,73 @@ class ReplicateObject extends BackbeatTask { const writeStartTime = Date.now(); return this.backbeatDest.send(putCommand, { abortSignal: abortController.signal }) .then(data => { + const { ExistingMicroVersionId } = data; + switch (ExistingMicroVersionId) { + case undefined: + case null: + break; // VersionId did not match on putData + case '': // Existing object with no microVersionId (cloudserver version pre crr cascade) + log.info('cascade putData: data at destination, ' + + 'old object without microVersionId, proceeding with putmetadata', + { + method: 'ReplicateObject._getAndPutPartOnce', + entry: destEntry.getLogInfo(), + }); + return doneOnce(null, partAlreadyAtDest); + default: { + // microVersionId provided, check its value to detect loop or stale events + // to determine if we need to update the metadata + const destinationMicroVersionId = decode(ExistingMicroVersionId); + if (destinationMicroVersionId instanceof Error) { + log.error('failed to decode ExistingMicroVersionId from putData', { + method: 'ReplicateObject._getAndPutPartOnce', + entry: destEntry.getLogInfo(), + error: destinationMicroVersionId.message, + }); + return doneOnce(destinationMicroVersionId); + } + const sourceMicroVersionId = sourceEntry.getMicroVersionId(); + if (!sourceMicroVersionId) { + log.info('cascade putData: data at destination, ' + + 'source has no microVersionId, proceeding with putmetadata', + { + method: 'ReplicateObject._getAndPutPartOnce', + entry: destEntry.getLogInfo(), + }); + return doneOnce(null, partAlreadyAtDest); + } + const event = checkCrrCascadeEvent(sourceMicroVersionId, destinationMicroVersionId); + if (event === 'loop') { + log.info('cascade loop detected on putData: ' + + 'destination already has this exact revision, skipping', + { + method: 'ReplicateObject._getAndPutPartOnce', + entry: destEntry.getLogInfo(), + }); + return doneOnce(cascadeLoopDetected); + } + if (event === 'stale') { + log.info('cascade stale on putData: ' + + 'destination already has a newer revision', + { + method: 'ReplicateObject._getAndPutPartOnce', + entry: destEntry.getLogInfo(), + }); + return doneOnce(cascadeDataComplete); + } + // proceed: source is newer, skip data write and update metadata + log.info('cascade putData: data already at destination, ' + + 'proceeding with metadata update', + { + method: 'ReplicateObject._getAndPutPartOnce', + entry: destEntry.getLogInfo(), + }); + return doneOnce(null, partAlreadyAtDest); + } + } + + // ExistingMicroVersionId is absent : + // data was freshly written at the returned Location. partObj.setDataLocation(data.Location[0]); // Set encryption parameters that were used to encrypt the @@ -641,10 +717,20 @@ class ReplicateObject extends BackbeatTask { // destination bucket has to be versioning enabled. VersioningRequired: true, RequestUids: log.getSerializedUids(), + MicroVersionId: entry.getMicroVersionId(), }); const writeStartTime = Date.now(); return this.backbeatDest.send(command) .then(data => { + if (data.MicroVersionIdExists) { + log.info('cascade loop detected on putMetadata: ' + + 'microVersionId already at destination', + { + method: 'ReplicateObject._putMetadataOnce', + entry: entry.getLogInfo(), + }); + return cbOnce(cascadeLoopDetected); + } this._publishMetadataWriteMetrics(mdBlob, writeStartTime); return cbOnce(null, data); }) @@ -654,6 +740,15 @@ class ReplicateObject extends BackbeatTask { if (err.ObjNotFound || err.name === 'ObjNotFound') { return cbOnce(err); } + if (err.$metadata?.httpStatusCode === 409) { + log.info('cascade stale on putMetadata: ' + + 'destination has a newer revision, marking COMPLETED', + { + method: 'ReplicateObject._putMetadataOnce', + entry: entry.getLogInfo(), + }); + return cbOnce(cascadeDataComplete); + } log.error('an error occurred when putting metadata to S3', { method: 'ReplicateObject._putMetadataOnce', @@ -889,13 +984,14 @@ class ReplicateObject extends BackbeatTask { return this._getAndPutData(sourceEntry, destEntry, log, next); } - return next(null, []); + return next(null, [], false); }, // update location, replication status and put metadata in // target bucket - (destLocations, next) => { + (destLocations, allPartsAlreadyAtDest, next) => { + const localMdOnly = mdOnly || allPartsAlreadyAtDest; destEntry.setLocation(destLocations); - this._putMetadata(destEntry, mdOnly, log, err => { + this._putMetadata(destEntry, localMdOnly, log, err => { if (err) { return this._deleteOrphans( destEntry, destLocations, log, () => next(err)); @@ -915,9 +1011,9 @@ class ReplicateObject extends BackbeatTask { next => this._getAndPutData(sourceEntry, destEntry, log, next), // update location, replication status and put metadata in // target bucket - (location, next) => { + (location, allPartsAlreadyAtDest, next) => { destEntry.setLocation(location); - this._putMetadata(destEntry, false, log, next); + this._putMetadata(destEntry, allPartsAlreadyAtDest, log, next); }, ], err => this._handleReplicationOutcome( err, sourceEntry, destEntry, kafkaEntry, log, done)); @@ -925,10 +1021,20 @@ class ReplicateObject extends BackbeatTask { _handleReplicationOutcome(err, sourceEntry, destEntry, kafkaEntry, log, done) { - if (!err) { - log.debug('replication succeeded for object, publishing ' + - 'replication status as COMPLETED', - { entry: sourceEntry.getLogInfo() }); + if (!err || err === cascadeLoopDetected || err === cascadeDataComplete) { + if (err === cascadeLoopDetected) { + log.info('replication completed via cascade loop: ' + + 'object already at destination with the same revision', + { entry: sourceEntry.getLogInfo() }); + } else if (err === cascadeDataComplete) { + log.info('replication completed: destination already holds ' + + 'this version with an equal or newer revision', + { entry: sourceEntry.getLogInfo() }); + } else { + log.debug('replication succeeded for object, publishing ' + + 'replication status as COMPLETED', + { entry: sourceEntry.getLogInfo() }); + } this._publishReplicationStatus( sourceEntry, 'COMPLETED', { kafkaEntry, log }); return done(null, { committable: false }); @@ -989,3 +1095,5 @@ class ReplicateObject extends BackbeatTask { } module.exports = ReplicateObject; +// Exported for tests only +module.exports._cascadeSignals = { cascadeLoopDetected, cascadeDataComplete, partAlreadyAtDest }; diff --git a/package.json b/package.json index 5d1b941d3..e6dadf8af 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "@aws-sdk/client-s3": "^3.921.0", "@aws-sdk/client-sts": "^3.921.0", "@aws-sdk/credential-providers": "^3.921.0", - "@scality/cloudserverclient": "^1.0.8", + "@scality/cloudserverclient": "file:../cloudserverclient/scality-cloudserverclient-v1.0.9.tgz", "@smithy/node-http-handler": "^3.3.3", "JSONStream": "^1.3.5", "arsenal": "git+https://github.com/scality/arsenal#8.3.9", diff --git a/tests/unit/replication/ReplicateObject.spec.js b/tests/unit/replication/ReplicateObject.spec.js index 038afe021..d0561a2c4 100644 --- a/tests/unit/replication/ReplicateObject.spec.js +++ b/tests/unit/replication/ReplicateObject.spec.js @@ -1,10 +1,14 @@ const assert = require('assert'); const sinon = require('sinon'); +const { Readable } = require('stream'); const QueueEntry = require('../../../lib/models/QueueEntry'); const ReplicateObject = require('../../../extensions/replication/tasks/ReplicateObject'); +const { _cascadeSignals } = ReplicateObject; const ClientManager = require('../../../lib/clients/ClientManager'); const locations = require('../../../conf/locationConfig.json'); +const { versioning } = require('arsenal'); +const { generateVersionId, encode } = versioning.VersionID; const { replicationEntry } = require('../../utils/kafkaEntries'); const fakeLogger = require('../../utils/fakeLogger'); @@ -72,6 +76,255 @@ describe('ReplicateObject', () => { sinon.restore(); }); + function makeMicroVersionIds() { + const older = generateVersionId('test', 'RG001'); + const newer = generateVersionId('test', 'RG001'); + return { older, newer, olderEncoded: encode(older), newerEncoded: encode(newer) }; + } + + function makeBodyStream() { + const stream = new Readable({ read() {} }); + process.nextTick(() => stream.push(null)); + return stream; + } + + function makeSourceEntry(microVersionId) { + return { + getBucket: () => 'src-bucket', + getObjectKey: () => 'key', + getEncodedVersionId: () => encode(generateVersionId('test', 'RG001')), + getMicroVersionId: () => microVersionId || null, + getOwnerId: () => 'canonical-id', + getLogInfo: () => ({}), + getLocation: () => [{ + key: 'data-key', size: 10, start: 0, + dataStoreName: 'file', dataStoreETag: '1:abc', + }], + getContentLength: () => 10, + }; + } + + function makeDestEntry() { + return { + getBucket: () => 'dest-bucket', + getObjectKey: () => 'key', + getOwnerId: () => 'canonical-id', + getLogInfo: () => ({}), + setAmzServerSideEncryption: () => {}, + setAmzEncryptionCustomerAlgorithm: () => {}, + setAmzEncryptionKeyId: () => {}, + setLocation: () => {}, + }; + } + + describe('putData : cascade ExistingMicroVersionId handling', () => { + let part; + + beforeEach(() => { + part = { key: 'data-key', size: 10, start: 0, + dataStoreName: 'file', dataStoreETag: '1:abc' }; + sinon.stub(task, '_publishReadMetrics').returns(); + sinon.stub(task, '_publishDataWriteMetrics').returns(); + }); + + function mockDataTransfer(destinationMicroVersionId) { + task.S3source = { + send: sinon.stub().resolves({ + Body: makeBodyStream(), + ContentLength: 10, + }), + }; + task.backbeatDest = { + send: sinon.stub().resolves({ ExistingMicroVersionId: destinationMicroVersionId }), + }; + } + + it('should return partAlreadyAtDest when ExistingMicroVersionId is empty (pre cascade objects)', done => { + const { older } = makeMicroVersionIds(); + const destinationMicroVersionId = ''; + mockDataTransfer(destinationMicroVersionId); + task._getAndPutPartOnce(makeSourceEntry(older), makeDestEntry(), part, fakeLogger, (err, result) => { + assert.ifError(err); + assert.strictEqual(result, _cascadeSignals.partAlreadyAtDest); + sinon.assert.notCalled(task._publishDataWriteMetrics); + done(); + }); + }); + + it('should return partAlreadyAtDest when ExistingMicroVersionId present but source has no microVersionId', done => { + const { olderEncoded } = makeMicroVersionIds(); + mockDataTransfer(olderEncoded); + task._getAndPutPartOnce(makeSourceEntry(null), makeDestEntry(), part, fakeLogger, (err, result) => { + assert.ifError(err); + assert.strictEqual(result, _cascadeSignals.partAlreadyAtDest); + sinon.assert.notCalled(task._publishDataWriteMetrics); + done(); + }); + }); + + it('should return cascadeLoopDetected when microVersionIds matches (loop)', done => { + const { older, olderEncoded } = makeMicroVersionIds(); + mockDataTransfer(olderEncoded); // destination has the same ID as source + task._getAndPutPartOnce(makeSourceEntry(older), makeDestEntry(), part, fakeLogger, (err, result) => { + assert.strictEqual(err, _cascadeSignals.cascadeLoopDetected); + assert.strictEqual(result, undefined); + done(); + }); + }); + + it('should return cascadeDataComplete when destination has newer microVersionId (stale)', done => { + const { older, newer, newerEncoded } = makeMicroVersionIds(); + mockDataTransfer(newerEncoded); + task._getAndPutPartOnce(makeSourceEntry(older), makeDestEntry(), part, fakeLogger, (err, result) => { + assert.strictEqual(err, _cascadeSignals.cascadeDataComplete); + assert.strictEqual(result, undefined); + done(); + }); + }); + + it('should return partAlreadyAtDest when source has newer microVersionId (proceed)', done => { + const { older, newer, olderEncoded } = makeMicroVersionIds(); + mockDataTransfer(olderEncoded); + task._getAndPutPartOnce(makeSourceEntry(newer), makeDestEntry(), part, fakeLogger, (err, result) => { + assert.ifError(err); + assert.strictEqual(result, _cascadeSignals.partAlreadyAtDest); + sinon.assert.notCalled(task._publishDataWriteMetrics); + done(); + }); + }); + + it('should propagate error when ExistingMicroVersionId cannot be decoded', done => { + const destinationMicroVersionId = 'not-a-valid-encoded-version-id-xxx'; + mockDataTransfer(destinationMicroVersionId); + task._getAndPutPartOnce(makeSourceEntry('some-mvid'), makeDestEntry(), part, fakeLogger, (err) => { + assert.ok(err instanceof Error, 'should propagate decode error'); + done(); + }); + }); + + it('should set data location and publish metrics when ExistingMicroVersionId is absent', done => { + task.S3source = { + send: sinon.stub().resolves({ Body: makeBodyStream(), ContentLength: 10 }), + }; + task.backbeatDest = { + send: sinon.stub().resolves({ + Location: [{ key: 'new-key', dataStoreName: 'file' }], + ExistingMicroVersionId: undefined, + }), + }; + task._getAndPutPartOnce(makeSourceEntry(), makeDestEntry(), part, fakeLogger, (err, result) => { + assert.ifError(err); + assert.deepStrictEqual(result, { + key: 'new-key', + start: 0, + size: 10, + dataStoreName: 'file', + dataStoreETag: '1:abc', + blockId: undefined, + }); + sinon.assert.calledOnce(task._publishDataWriteMetrics); + done(); + }); + }); + }); + + describe('putMetadata : cascade handling', () => { + let entry; + + beforeEach(() => { + entry = QueueEntry.createFromKafkaEntry(replicationEntry); + task.targetRole = 'arn:aws:iam::123456789012:role/crr-role'; + }); + + it('should return cascadeLoopDetected and skip metrics when MicroVersionIdExists', done => { + const metricsStub = sinon.stub(task, '_publishMetadataWriteMetrics').returns(); + task.backbeatDest = { send: sinon.stub().resolves({ MicroVersionIdExists: true }) }; + task._putMetadataOnce(entry, false, fakeLogger, (err) => { + assert.strictEqual(err, _cascadeSignals.cascadeLoopDetected); + sinon.assert.notCalled(metricsStub); + done(); + }); + }); + + it('should return cascadeDataComplete on 409 stale response', done => { + sinon.stub(task, '_publishMetadataWriteMetrics').returns(); + const err409 = Object.assign(new Error('Conflict'), { + $metadata: { httpStatusCode: 409 }, + origin: 'target', + }); + task.backbeatDest = { send: sinon.stub().rejects(err409) }; + task._putMetadataOnce(entry, false, fakeLogger, (err) => { + assert.strictEqual(err, _cascadeSignals.cascadeDataComplete); + done(); + }); + }); + + it('should publish metrics and succeed on normal response', done => { + const metricsStub = sinon.stub(task, '_publishMetadataWriteMetrics').returns(); + task.backbeatDest = { send: sinon.stub().resolves({}) }; + task._putMetadataOnce(entry, false, fakeLogger, (err) => { + assert.ifError(err); + sinon.assert.calledOnce(metricsStub); + done(); + }); + }); + }); + + describe('_handleReplicationOutcome : cascade outcomes', () => { + let sourceEntry, destEntry, kafkaEntry; + + beforeEach(() => { + sourceEntry = QueueEntry.createFromKafkaEntry(replicationEntry); + destEntry = makeDestEntry(); + kafkaEntry = {}; + sinon.stub(task, '_publishReplicationStatus').returns(); + + sinon.stub(sourceEntry, 'toCompletedEntry').returns(sourceEntry); + sinon.stub(sourceEntry, 'toFailedEntry').returns(sourceEntry); + sinon.stub(sourceEntry, 'setReplicationSiteDataStoreVersionId').returns(sourceEntry); + sinon.stub(sourceEntry, 'getReplicationSiteDataStoreVersionId').returns('v1'); + }); + + it('should mark COMPLETED for cascadeLoopDetected', done => { + task._handleReplicationOutcome( + _cascadeSignals.cascadeLoopDetected, + sourceEntry, destEntry, kafkaEntry, fakeLogger, () => { + sinon.assert.calledWith(task._publishReplicationStatus, + sourceEntry, 'COMPLETED', sinon.match.any); + done(); + }); + }); + + it('should mark COMPLETED for cascadeDataComplete', done => { + task._handleReplicationOutcome( + _cascadeSignals.cascadeDataComplete, + sourceEntry, destEntry, kafkaEntry, fakeLogger, () => { + sinon.assert.calledWith(task._publishReplicationStatus, + sourceEntry, 'COMPLETED', sinon.match.any); + done(); + }); + }); + + it('should mark COMPLETED on successful replication', done => { + task._handleReplicationOutcome( + null, sourceEntry, destEntry, kafkaEntry, fakeLogger, () => { + sinon.assert.calledWith(task._publishReplicationStatus, + sourceEntry, 'COMPLETED', sinon.match.any); + done(); + }); + }); + + it('should mark FAILED for real errors', done => { + const realErr = Object.assign(new Error('network failure'), { origin: 'target' }); + task._handleReplicationOutcome( + realErr, sourceEntry, destEntry, kafkaEntry, fakeLogger, () => { + sinon.assert.calledWith(task._publishReplicationStatus, + sourceEntry, 'FAILED', sinon.match.any); + done(); + }); + }); + }); + describe('_setTargetAccountMd', () => { it('should skip gettin target account info when auth type is assumeRole', done => { sinon.stub(task, '_setupDestClients').returns(); diff --git a/yarn.lock b/yarn.lock index aa740f23e..149cdc462 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1203,6 +1203,16 @@ "@smithy/types" "^4.12.0" tslib "^2.6.2" +"@aws-sdk/middleware-expect-continue@^3.972.8": + version "3.972.14" + resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.14.tgz#6bffa547da965dba80ee123ac1f8f1446cc596c2" + integrity sha512-3TNFEVGO4sWZj9TEXOCZLzGEctXHnaO4fk2EQ8KVaboTbwHmEPEQrm17Xb9koImUIXEw0sgi2xtHjg7LuTS3rA== + dependencies: + "@aws-sdk/types" "^3.973.9" + "@smithy/core" "^3.24.5" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/middleware-expect-continue@^3.972.9": version "3.972.9" resolved "https://registry.yarnpkg.com/@aws-sdk/middleware-expect-continue/-/middleware-expect-continue-3.972.9.tgz#ad62cbc4c5f310a5d104b7fc1150eca13a3c07a4" @@ -1824,6 +1834,14 @@ "@smithy/types" "^4.14.0" tslib "^2.6.2" +"@aws-sdk/types@^3.973.9": + version "3.973.9" + resolved "https://registry.yarnpkg.com/@aws-sdk/types/-/types-3.973.9.tgz#7d1c08cc6e82ec2ac2f2da102a7dd55806592f7f" + integrity sha512-kuBfgQVdcz5Bmapc4A13YbpVw/pXkesfhetcFYwbntqas8sF41OHyd4o28+/TG2ZQdHBsv90Lsu5y6oitvYCdg== + dependencies: + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@aws-sdk/util-arn-parser@3.893.0": version "3.893.0" resolved "https://registry.yarnpkg.com/@aws-sdk/util-arn-parser/-/util-arn-parser-3.893.0.tgz#fcc9b792744b9da597662891c2422dda83881d8d" @@ -2757,6 +2775,15 @@ JSONStream "^1.3.5" fast-xml-parser "^5.5.7" +"@scality/cloudserverclient@file:../cloudserverclient/scality-cloudserverclient-v1.0.9.tgz": + version "1.0.9" + resolved "file:../cloudserverclient/scality-cloudserverclient-v1.0.9.tgz#41b761c46e0e620abbb00cc3fe2844b2e8fc84b8" + dependencies: + "@aws-sdk/client-s3" "^3.1009.0" + "@aws-sdk/middleware-expect-continue" "^3.972.8" + JSONStream "^1.3.5" + fast-xml-parser "^5.5.7" + "@scality/hdclient@^1.3.1": version "1.3.1" resolved "https://registry.yarnpkg.com/@scality/hdclient/-/hdclient-1.3.1.tgz#52f7d8e9051278f7d610de30828aac84c3943498" @@ -2977,6 +3004,15 @@ "@smithy/uuid" "^1.1.2" tslib "^2.6.2" +"@smithy/core@^3.24.5": + version "3.24.5" + resolved "https://registry.yarnpkg.com/@smithy/core/-/core-3.24.5.tgz#396ca5662afc6d83a8f41b7e492e427c48a0924e" + integrity sha512-Kt8phUg45M15EjhYAbZ+fFikYneijLu9Liugz8ZsYz2i8j0hzGv27LWKpEHYRfvj+LyCOSijpcR/2i8RouV+cA== + dependencies: + "@aws-crypto/crc32" "5.2.0" + "@smithy/types" "^4.14.2" + tslib "^2.6.2" + "@smithy/credential-provider-imds@^4.2.13": version "4.2.13" resolved "https://registry.yarnpkg.com/@smithy/credential-provider-imds/-/credential-provider-imds-4.2.13.tgz#c0533f362dec6644f403c7789d8e81233f78c63f" @@ -3940,6 +3976,13 @@ dependencies: tslib "^2.6.2" +"@smithy/types@^4.14.2": + version "4.14.2" + resolved "https://registry.yarnpkg.com/@smithy/types/-/types-4.14.2.tgz#6034ff1e0e52bfb7d744ac371b651a8bf21f30f1" + integrity sha512-P+otAxbV4CqBybp7EkcJCrig63yE2E7PuNVOmilVMRcx/O+QDzGULTrKsq4DV13gSfak9ObPrWaHl/9bL5YcWw== + dependencies: + tslib "^2.6.2" + "@smithy/types@^4.9.0": version "4.9.0" resolved "https://registry.yarnpkg.com/@smithy/types/-/types-4.9.0.tgz#c6636ddfa142e1ddcb6e4cf5f3e1a628d420486f"