diff --git a/lib/api/apiUtils/object/bumpMicroVersionId.js b/lib/api/apiUtils/object/bumpMicroVersionId.js
new file mode 100644
index 0000000000..1a76970726
--- /dev/null
+++ b/lib/api/apiUtils/object/bumpMicroVersionId.js
@@ -0,0 +1,27 @@
+const { versioning } = require('arsenal');
+const { config } = require('../../../Config');
+
+/**
+ * Bump objectMD.microVersionId. microVersionId is a generic
+ * metadata-revision marker, not a CRR-specific field, but cascaded CRR
+ * is its only consumer today - so we gate on replicationInfo to avoid
+ * inflating storage on objects that wouldn't use it. The gate can be
+ * widened later if another consumer needs it on every object.
+ * Pass `force = true` to bump unconditionally.
+ *
+ * @param {object} objectMD - object MD POJO or `md.getValue()`
+ * @param {boolean} [force] - bump even without replicationInfo
+ * @return {undefined}
+ */
+function bumpMicroVersionId(objectMD, force) {
+ if (!force && !objectMD?.replicationInfo) {
+ return;
+ }
+
+ const { instanceId, replicationGroupId } = config;
+
+ // eslint-disable-next-line no-param-reassign
+ objectMD.microVersionId = versioning.VersionID.generateVersionId(instanceId, replicationGroupId);
+}
+
+module.exports = bumpMicroVersionId;
diff --git a/lib/api/apiUtils/object/getReplicationInfo.js b/lib/api/apiUtils/object/getReplicationInfo.js
index b7e4afb25d..c02a260ee2 100644
--- a/lib/api/apiUtils/object/getReplicationInfo.js
+++ b/lib/api/apiUtils/object/getReplicationInfo.js
@@ -1,5 +1,4 @@
-const { isServiceAccount, getServiceAccountProperties } =
- require('../authorization/permissionChecks');
+const { isServiceAccount, getServiceAccountProperties } = require('../authorization/permissionChecks');
const { replicationBackends } = require('arsenal').constants;
function _getBackend(objectMD, site) {
@@ -23,15 +22,13 @@ function _getStorageClasses(s3config, rule) {
const { replicationEndpoints } = s3config;
// If no storage class, use the given default endpoint or the sole endpoint
if (replicationEndpoints.length > 0) {
- const endPoint =
- replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
+ const endPoint = replicationEndpoints.find(endpoint => endpoint.default) || replicationEndpoints[0];
return [endPoint.site];
}
return undefined;
}
-function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType,
- objectMD, bucketMD) {
+function _getReplicationInfo(s3config, rule, replicationConfig, content, operationType, objectMD, bucketMD) {
const storageTypes = [];
const backends = [];
const storageClasses = _getStorageClasses(s3config, rule);
@@ -39,9 +36,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati
return undefined;
}
storageClasses.forEach(storageClass => {
- const storageClassName =
- storageClass.endsWith(':preferred_read') ?
- storageClass.split(':')[0] : storageClass;
+ const storageClassName = storageClass.endsWith(':preferred_read') ? storageClass.split(':')[0] : storageClass;
// TODO CLDSRV-646: for consistency, should we look at replicationEndpoints instead, like
// `_getStorageClasses()` ?
const location = s3config.locationConstraints[storageClassName];
@@ -62,6 +57,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati
role: replicationConfig.role,
storageType: storageTypes.join(','),
isNFS: bucketMD.isNFS(),
+ isReplica: false,
};
}
@@ -80,8 +76,7 @@ function _getReplicationInfo(s3config, rule, replicationConfig, content, operati
* @param {AuthInfo} [authInfo] - authentication info of object owner
* @return {undefined}
*/
-function getReplicationInfo(
- s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) {
+function getReplicationInfo(s3config, objKey, bucketMD, isMD, objSize, operationType, objectMD, authInfo) {
const content = isMD || objSize === 0 ? ['METADATA'] : ['DATA', 'METADATA'];
const config = bucketMD.getReplicationConfiguration();
@@ -106,17 +101,14 @@ function getReplicationInfo(
if (!authInfo || !isServiceAccount(authInfo.getCanonicalID())) {
doReplicate = true;
} else {
- const serviceAccountProps = getServiceAccountProperties(
- authInfo.getCanonicalID());
+ const serviceAccountProps = getServiceAccountProperties(authInfo.getCanonicalID());
doReplicate = serviceAccountProps.canReplicate;
}
if (doReplicate) {
- const rule = config.rules.find(
- rule => (objKey.startsWith(rule.prefix) && rule.enabled));
+ const rule = config.rules.find(rule => objKey.startsWith(rule.prefix) && rule.enabled);
if (rule) {
// TODO CLDSRV-646 : should "merge" the replicationInfo for different rules
- return _getReplicationInfo(
- s3config, rule, config, content, operationType, objectMD, bucketMD);
+ return _getReplicationInfo(s3config, rule, config, content, operationType, objectMD, bucketMD);
}
}
}
diff --git a/lib/api/apiUtils/object/versioning.js b/lib/api/apiUtils/object/versioning.js
index 418224a4a3..1ece12c652 100644
--- a/lib/api/apiUtils/object/versioning.js
+++ b/lib/api/apiUtils/object/versioning.js
@@ -10,8 +10,7 @@ const { scaledMsPerDay } = config.getTimeOptions();
const versionIdUtils = versioning.VersionID;
// Use Arsenal function to generate a version ID used internally by metadata
// for null versions that are created before bucket versioning is configured
-const nonVersionedObjId =
- versionIdUtils.getInfVid(config.replicationGroupId);
+const nonVersionedObjId = versionIdUtils.getInfVid(config.replicationGroupId);
/** decodeVID - decode the version id
* @param {string} versionId - version ID
@@ -101,25 +100,24 @@ function _storeNullVersionMD(bucketName, objKey, nullVersionId, objMD, log, cb)
nullVersionMD.originOp = 's3:StoreNullVersion';
metadata.putObjectMD(bucketName, objKey, nullVersionMD, { versionId }, log, err => {
if (err) {
- log.debug('error from metadata storing null version as new version',
- { error: err });
+ log.debug('error from metadata storing null version as new version', { error: err });
}
-
+
cb(err);
});
}
/** check existence and get location of null version data for deletion
-* @param {string} bucketName - name of bucket
-* @param {string} objKey - name of object key
-* @param {object} options - metadata options for getting object MD
-* @param {string} options.versionId - version to get from metadata
-* @param {object} mst - info about the master version
-* @param {string} mst.versionId - the master version's version id
-* @param {RequestLogger} log - logger instanceof
-* @param {function} cb - callback
-* @return {undefined} - and call callback with (err, dataToDelete)
-*/
+ * @param {string} bucketName - name of bucket
+ * @param {string} objKey - name of object key
+ * @param {object} options - metadata options for getting object MD
+ * @param {string} options.versionId - version to get from metadata
+ * @param {object} mst - info about the master version
+ * @param {string} mst.versionId - the master version's version id
+ * @param {RequestLogger} log - logger instanceof
+ * @param {function} cb - callback
+ * @return {undefined} - and call callback with (err, dataToDelete)
+ */
function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb) {
const nullOptions = {};
if (!options.deleteData) {
@@ -135,38 +133,40 @@ function _prepareNullVersionDeletion(bucketName, objKey, options, mst, log, cb)
// PUT via this option
nullOptions.deleteNullKey = true;
}
- return metadata.getObjectMD(bucketName, objKey, options, log,
- (err, versionMD) => {
- if (err) {
- // the null key may not exist, hence it's a normal
- // situation to have a NoSuchKey error, in which case
- // there is nothing to delete
- if (err.is.NoSuchKey) {
- log.debug('null version does not exist', {
- method: '_prepareNullVersionDeletion',
- });
- } else {
- log.warn('could not get null version metadata', {
- error: err,
- method: '_prepareNullVersionDeletion',
- });
- }
- return cb(err);
- }
- if (versionMD.location) {
- const dataToDelete = Array.isArray(versionMD.location) ?
- versionMD.location : [versionMD.location];
- nullOptions.dataToDelete = dataToDelete;
+ return metadata.getObjectMD(bucketName, objKey, options, log, (err, versionMD) => {
+ if (err) {
+ // the null key may not exist, hence it's a normal
+ // situation to have a NoSuchKey error, in which case
+ // there is nothing to delete
+ if (err.is.NoSuchKey) {
+ log.debug('null version does not exist', {
+ method: '_prepareNullVersionDeletion',
+ });
+ } else {
+ log.warn('could not get null version metadata', {
+ error: err,
+ method: '_prepareNullVersionDeletion',
+ });
}
- return cb(null, nullOptions);
- });
+ return cb(err);
+ }
+ if (versionMD.location) {
+ const dataToDelete = Array.isArray(versionMD.location) ? versionMD.location : [versionMD.location];
+ nullOptions.dataToDelete = dataToDelete;
+ }
+ return cb(null, nullOptions);
+ });
}
function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
return metadata.deleteObjectMD(bucketName, objKey, options, log, err => {
if (err) {
- log.warn('metadata error deleting null versioned key',
- { bucketName, objKey, error: err, method: '_deleteNullVersionMD' });
+ log.warn('metadata error deleting null versioned key', {
+ bucketName,
+ objKey,
+ error: err,
+ method: '_deleteNullVersionMD',
+ });
}
return cb(err);
});
@@ -193,7 +193,7 @@ function _deleteNullVersionMD(bucketName, objKey, options, log, cb) {
version key, if needed
*/
function processVersioningState(mst, vstat, nullVersionCompatMode) {
- const versioningSuspended = (vstat === 'Suspended');
+ const versioningSuspended = vstat === 'Suspended';
const masterIsNull = mst.exists && (mst.isNull || !mst.versionId);
if (versioningSuspended) {
@@ -244,7 +244,7 @@ function processVersioningState(mst, vstat, nullVersionCompatMode) {
if (masterIsNull) {
// if master is a null version or a non-versioned key,
// copy it to a new null key
- const nullVersionId = (mst.isNull && mst.versionId) ? mst.versionId : nonVersionedObjId;
+ const nullVersionId = mst.isNull && mst.versionId ? mst.versionId : nonVersionedObjId;
if (nullVersionCompatMode) {
options.extraMD = {
nullVersionId,
@@ -311,8 +311,7 @@ function getMasterState(objMD) {
};
if (objMD.location) {
- mst.objLocation = Array.isArray(objMD.location) ?
- objMD.location : [objMD.location];
+ mst.objLocation = Array.isArray(objMD.location) ? objMD.location : [objMD.location];
}
return mst;
@@ -332,8 +331,7 @@ function getMasterState(objMD) {
* options.versioning - (true/undefined) metadata instruction to create new ver
* options.isNull - (true/undefined) whether new version is null or not
*/
-function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
- log, callback) {
+function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD, log, callback) {
const mst = getMasterState(objMD);
const vCfg = bucketMD.getVersioningConfiguration();
@@ -342,50 +340,57 @@ function versioningPreprocessing(bucketName, bucketMD, objectKey, objMD,
return process.nextTick(callback, null, options);
}
- const { options, nullVersionId, delOptions } =
- processVersioningState(mst, vCfg.Status, config.nullVersionCompatMode);
-
- return async.series([
- function storeNullVersionMD(next) {
- if (!nullVersionId) {
- return process.nextTick(next);
- }
+ const { options, nullVersionId, delOptions } = processVersioningState(
+ mst,
+ vCfg.Status,
+ config.nullVersionCompatMode,
+ );
+
+ return async.series(
+ [
+ function storeNullVersionMD(next) {
+ if (!nullVersionId) {
+ return process.nextTick(next);
+ }
- options.nullVersionId = nullVersionId;
- return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
- },
- function prepareNullVersionDeletion(next) {
- if (!delOptions) {
- return process.nextTick(next);
- }
- return _prepareNullVersionDeletion(
- bucketName, objectKey, delOptions, mst, log,
- (err, nullOptions) => {
+ options.nullVersionId = nullVersionId;
+ return _storeNullVersionMD(bucketName, objectKey, nullVersionId, objMD, log, next);
+ },
+ function prepareNullVersionDeletion(next) {
+ if (!delOptions) {
+ return process.nextTick(next);
+ }
+ return _prepareNullVersionDeletion(bucketName, objectKey, delOptions, mst, log, (err, nullOptions) => {
if (err) {
return next(err);
}
Object.assign(options, nullOptions);
return next();
});
- },
- function deleteNullVersionMD(next) {
- if (delOptions &&
- delOptions.versionId &&
- delOptions.versionId !== 'null') {
- // backward-compat: delete old null versioned key
- return _deleteNullVersionMD(
- bucketName, objectKey, { versionId: delOptions.versionId, overheadField }, log, next);
+ },
+ function deleteNullVersionMD(next) {
+ if (delOptions && delOptions.versionId && delOptions.versionId !== 'null') {
+ // backward-compat: delete old null versioned key
+ return _deleteNullVersionMD(
+ bucketName,
+ objectKey,
+ { versionId: delOptions.versionId, overheadField },
+ log,
+ next,
+ );
+ }
+ return process.nextTick(next);
+ },
+ ],
+ err => {
+ // it's possible there was a prior request that deleted the
+ // null version, so proceed with putting a new version
+ if (err && err.is.NoSuchKey) {
+ return callback(null, options);
}
- return process.nextTick(next);
+ return callback(err, options);
},
- ], err => {
- // it's possible there was a prior request that deleted the
- // null version, so proceed with putting a new version
- if (err && err.is.NoSuchKey) {
- return callback(null, options);
- }
- return callback(err, options);
- });
+ );
}
/** Return options to pass to Metadata layer for version-specific
@@ -545,7 +550,7 @@ function overwritingVersioning(objMD, metadataStoreParams) {
restoreRequestedAt: objMD.archive?.restoreRequestedAt,
restoreRequestedDays: objMD.archive?.restoreRequestedDays,
restoreCompletedAt: new Date(now),
- restoreWillExpireAt: new Date(now + (days * scaledMsPerDay)),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
};
/* eslint-enable no-param-reassign */
diff --git a/lib/api/objectDeleteTagging.js b/lib/api/objectDeleteTagging.js
index 71115ffe5a..9c8420c737 100644
--- a/lib/api/objectDeleteTagging.js
+++ b/lib/api/objectDeleteTagging.js
@@ -1,8 +1,11 @@
const async = require('async');
const { errors } = require('arsenal');
-const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions }
- = require('./apiUtils/object/versioning');
+const {
+ decodeVersionId,
+ getVersionIdResHeader,
+ getVersionSpecificMetadataOptions,
+} = require('./apiUtils/object/versioning');
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
@@ -10,6 +13,7 @@ const monitoring = require('../utilities/monitoringHandler');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
+const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId');
const { data } = require('../data/wrapper');
const { config } = require('../Config');
const REPLICATION_ACTION = 'DELETE_TAGGING';
@@ -48,75 +52,82 @@ function objectDeleteTagging(authInfo, request, log, callback) {
request,
};
- return async.waterfall([
- next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log,
- (err, bucket, objectMD) => {
- if (err) {
- log.trace('request authorization failed',
- { method: 'objectDeleteTagging', error: err });
- return next(err);
- }
- if (!objectMD) {
- const err = reqVersionId ? errors.NoSuchVersion :
- errors.NoSuchKey;
- log.trace('error no object metadata found',
- { method: 'objectDeleteTagging', error: err });
- return next(err, bucket);
- }
- if (objectMD.isDeleteMarker) {
- log.trace('version is a delete marker',
- { method: 'objectDeleteTagging' });
- // FIXME we should return a `x-amz-delete-marker: true` header,
- // see S3C-7592
- return next(errors.MethodNotAllowed, bucket);
- }
- return next(null, bucket, objectMD);
- }),
- (bucket, objectMD, next) => {
- // eslint-disable-next-line no-param-reassign
- objectMD.tags = {};
- const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
- const replicationInfo = getReplicationInfo(config,
- objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
- if (replicationInfo) {
+ return async.waterfall(
+ [
+ next =>
+ standardMetadataValidateBucketAndObj(
+ metadataValParams,
+ request.actionImplicitDenies,
+ log,
+ (err, bucket, objectMD) => {
+ if (err) {
+ log.trace('request authorization failed', { method: 'objectDeleteTagging', error: err });
+ return next(err);
+ }
+ if (!objectMD) {
+ const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
+ log.trace('error no object metadata found', { method: 'objectDeleteTagging', error: err });
+ return next(err, bucket);
+ }
+ if (objectMD.isDeleteMarker) {
+ log.trace('version is a delete marker', { method: 'objectDeleteTagging' });
+ // FIXME we should return a `x-amz-delete-marker: true` header,
+ // see S3C-7592
+ return next(errors.MethodNotAllowed, bucket);
+ }
+ return next(null, bucket, objectMD);
+ },
+ ),
+ (bucket, objectMD, next) => {
// eslint-disable-next-line no-param-reassign
- objectMD.replicationInfo = Object.assign({},
- objectMD.replicationInfo, replicationInfo);
+ objectMD.tags = {};
+ const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
+ const replicationInfo = getReplicationInfo(
+ config,
+ objectKey,
+ bucket,
+ true,
+ 0,
+ REPLICATION_ACTION,
+ objectMD,
+ );
+ if (replicationInfo) {
+ // eslint-disable-next-line no-param-reassign
+ objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo);
+ bumpMicroVersionId(objectMD);
+ }
+ // eslint-disable-next-line no-param-reassign
+ objectMD.originOp = 's3:ObjectTagging:Delete';
+ metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err =>
+ next(err, bucket, objectMD),
+ );
+ },
+ (bucket, objectMD, next) =>
+ // if external backends handles tagging
+ data.objectTagging('Delete', objectKey, bucket.getName(), objectMD, log, err =>
+ next(err, bucket, objectMD),
+ ),
+ ],
+ (err, bucket, objectMD) => {
+ const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket);
+ if (err) {
+ log.trace('error processing request', { error: err, method: 'objectDeleteTagging' });
+ monitoring.promMetrics('DELETE', bucketName, err.code, 'deleteObjectTagging');
+ } else {
+ pushMetric('deleteObjectTagging', log, {
+ authInfo,
+ bucket: bucketName,
+ keys: [objectKey],
+ versionId: objectMD ? objectMD.versionId : undefined,
+ location: objectMD ? objectMD.dataStoreName : undefined,
+ });
+ monitoring.promMetrics('DELETE', bucketName, '200', 'deleteObjectTagging');
+ const verCfg = bucket.getVersioningConfiguration();
+ additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD);
}
- // eslint-disable-next-line no-param-reassign
- objectMD.originOp = 's3:ObjectTagging:Delete';
- metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
- log, err =>
- next(err, bucket, objectMD));
+ return callback(err, additionalResHeaders);
},
- (bucket, objectMD, next) =>
- // if external backends handles tagging
- data.objectTagging('Delete', objectKey, bucket.getName(), objectMD,
- log, err => next(err, bucket, objectMD)),
- ], (err, bucket, objectMD) => {
- const additionalResHeaders = collectCorsHeaders(request.headers.origin,
- request.method, bucket);
- if (err) {
- log.trace('error processing request', { error: err,
- method: 'objectDeleteTagging' });
- monitoring.promMetrics(
- 'DELETE', bucketName, err.code, 'deleteObjectTagging');
- } else {
- pushMetric('deleteObjectTagging', log, {
- authInfo,
- bucket: bucketName,
- keys: [objectKey],
- versionId: objectMD ? objectMD.versionId : undefined,
- location: objectMD ? objectMD.dataStoreName : undefined,
- });
- monitoring.promMetrics(
- 'DELETE', bucketName, '200', 'deleteObjectTagging');
- const verCfg = bucket.getVersioningConfiguration();
- additionalResHeaders['x-amz-version-id'] =
- getVersionIdResHeader(verCfg, objectMD);
- }
- return callback(err, additionalResHeaders);
- });
+ );
}
module.exports = objectDeleteTagging;
diff --git a/lib/api/objectPutLegalHold.js b/lib/api/objectPutLegalHold.js
index c16f2c84e8..5fe844a1e8 100644
--- a/lib/api/objectPutLegalHold.js
+++ b/lib/api/objectPutLegalHold.js
@@ -2,9 +2,13 @@ const async = require('async');
const { errors, errorInstances, s3middleware } = require('arsenal');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
-const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } =
- require('./apiUtils/object/versioning');
+const {
+ decodeVersionId,
+ getVersionIdResHeader,
+ getVersionSpecificMetadataOptions,
+} = require('./apiUtils/object/versioning');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
+const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId');
const metadata = require('../metadata/wrapper');
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
@@ -47,78 +51,88 @@ function objectPutLegalHold(authInfo, request, log, callback) {
request,
};
- return async.waterfall([
- next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log,
+ return async.waterfall(
+ [
+ next =>
+ standardMetadataValidateBucketAndObj(
+ metadataValParams,
+ request.actionImplicitDenies,
+ log,
+ (err, bucket, objectMD) => {
+ if (err) {
+ log.trace('request authorization failed', { method: 'objectPutLegalHold', error: err });
+ return next(err);
+ }
+ if (!objectMD) {
+ const err = versionId ? errors.NoSuchVersion : errors.NoSuchKey;
+ log.trace('error no object metadata found', { method: 'objectPutLegalHold', error: err });
+ return next(err, bucket);
+ }
+ if (objectMD.isDeleteMarker) {
+ log.trace('version is a delete marker', { method: 'objectPutLegalHold' });
+ // FIXME we should return a `x-amz-delete-marker: true` header,
+ // see S3C-7592
+ return next(errors.MethodNotAllowed, bucket);
+ }
+ if (!bucket.isObjectLockEnabled()) {
+ log.trace('object lock not enabled on bucket', { method: 'objectPutLegalHold' });
+ return next(
+ errorInstances.InvalidRequest.customizeDescription(
+ 'Bucket is missing Object Lock Configuration',
+ ),
+ bucket,
+ );
+ }
+ return next(null, bucket, objectMD);
+ },
+ ),
+ (bucket, objectMD, next) => {
+ log.trace('parsing legal hold');
+ parseLegalHoldXml(request.post, log, (err, res) => next(err, bucket, res, objectMD));
+ },
+ (bucket, legalHold, objectMD, next) => {
+ // eslint-disable-next-line no-param-reassign
+ objectMD.legalHold = legalHold;
+ const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
+ const replicationInfo = getReplicationInfo(
+ config,
+ objectKey,
+ bucket,
+ true,
+ 0,
+ REPLICATION_ACTION,
+ objectMD,
+ );
+ if (replicationInfo) {
+ // eslint-disable-next-line no-param-reassign
+ objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo);
+ bumpMicroVersionId(objectMD);
+ }
+ // eslint-disable-next-line no-param-reassign
+ objectMD.originOp = 's3:ObjectLegalHold:Put';
+ metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err =>
+ next(err, bucket, objectMD),
+ );
+ },
+ ],
(err, bucket, objectMD) => {
+ const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket);
if (err) {
- log.trace('request authorization failed',
- { method: 'objectPutLegalHold', error: err });
- return next(err);
- }
- if (!objectMD) {
- const err = versionId ? errors.NoSuchVersion :
- errors.NoSuchKey;
- log.trace('error no object metadata found',
- { method: 'objectPutLegalHold', error: err });
- return next(err, bucket);
- }
- if (objectMD.isDeleteMarker) {
- log.trace('version is a delete marker',
- { method: 'objectPutLegalHold' });
- // FIXME we should return a `x-amz-delete-marker: true` header,
- // see S3C-7592
- return next(errors.MethodNotAllowed, bucket);
- }
- if (!bucket.isObjectLockEnabled()) {
- log.trace('object lock not enabled on bucket',
- { method: 'objectPutLegalHold' });
- return next(errorInstances.InvalidRequest.customizeDescription(
- 'Bucket is missing Object Lock Configuration'
- ), bucket);
- }
- return next(null, bucket, objectMD);
- }),
- (bucket, objectMD, next) => {
- log.trace('parsing legal hold');
- parseLegalHoldXml(request.post, log, (err, res) =>
- next(err, bucket, res, objectMD));
- },
- (bucket, legalHold, objectMD, next) => {
- // eslint-disable-next-line no-param-reassign
- objectMD.legalHold = legalHold;
- const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
- const replicationInfo = getReplicationInfo(config,
- objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
- if (replicationInfo) {
- // eslint-disable-next-line no-param-reassign
- objectMD.replicationInfo = Object.assign({},
- objectMD.replicationInfo, replicationInfo);
+ log.trace('error processing request', { error: err, method: 'objectPutLegalHold' });
+ } else {
+ pushMetric('putObjectLegalHold', log, {
+ authInfo,
+ bucket: bucketName,
+ keys: [objectKey],
+ versionId: objectMD ? objectMD.versionId : undefined,
+ location: objectMD ? objectMD.dataStoreName : undefined,
+ });
+ const verCfg = bucket.getVersioningConfiguration();
+ additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD);
}
- // eslint-disable-next-line no-param-reassign
- objectMD.originOp = 's3:ObjectLegalHold:Put';
- metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
- log, err => next(err, bucket, objectMD));
+ return callback(err, additionalResHeaders);
},
- ], (err, bucket, objectMD) => {
- const additionalResHeaders = collectCorsHeaders(request.headers.origin,
- request.method, bucket);
- if (err) {
- log.trace('error processing request',
- { error: err, method: 'objectPutLegalHold' });
- } else {
- pushMetric('putObjectLegalHold', log, {
- authInfo,
- bucket: bucketName,
- keys: [objectKey],
- versionId: objectMD ? objectMD.versionId : undefined,
- location: objectMD ? objectMD.dataStoreName : undefined,
- });
- const verCfg = bucket.getVersioningConfiguration();
- additionalResHeaders['x-amz-version-id'] =
- getVersionIdResHeader(verCfg, objectMD);
- }
- return callback(err, additionalResHeaders);
- });
+ );
}
module.exports = objectPutLegalHold;
diff --git a/lib/api/objectPutRetention.js b/lib/api/objectPutRetention.js
index 6a7a2c8441..e281b71f91 100644
--- a/lib/api/objectPutRetention.js
+++ b/lib/api/objectPutRetention.js
@@ -1,13 +1,16 @@
const async = require('async');
const { errors, errorInstances, s3middleware } = require('arsenal');
-const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } =
- require('./apiUtils/object/versioning');
-const { ObjectLockInfo, hasGovernanceBypassHeader } =
- require('./apiUtils/object/objectLockHelpers');
+const {
+ decodeVersionId,
+ getVersionIdResHeader,
+ getVersionSpecificMetadataOptions,
+} = require('./apiUtils/object/versioning');
+const { ObjectLockInfo, hasGovernanceBypassHeader } = require('./apiUtils/object/objectLockHelpers');
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
+const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper');
const { config } = require('../Config');
@@ -50,99 +53,107 @@ function objectPutRetention(authInfo, request, log, callback) {
const hasGovernanceBypass = hasGovernanceBypassHeader(request.headers);
- return async.waterfall([
- next => {
- log.trace('parsing retention information');
- parseRetentionXml(request.post, log,
- (err, retentionInfo) => {
+ return async.waterfall(
+ [
+ next => {
+ log.trace('parsing retention information');
+ parseRetentionXml(request.post, log, (err, retentionInfo) => {
if (err) {
- log.trace('error parsing retention information',
- { error: err });
+ log.trace('error parsing retention information', { error: err });
return next(err);
}
- const remainingDays = Math.ceil(
- (new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24));
+ const remainingDays = Math.ceil((new Date(retentionInfo.date) - Date.now()) / (1000 * 3600 * 24));
metadataValParams.request.objectLockRetentionDays = remainingDays;
return next(null, retentionInfo);
});
- },
- (retentionInfo, next) => standardMetadataValidateBucketAndObj(metadataValParams,
- request.actionImplicitDenies, log, (err, bucket, objectMD) => {
- if (err) {
- log.trace('request authorization failed',
- { method: 'objectPutRetention', error: err });
- return next(err);
- }
- if (!objectMD) {
- const err = reqVersionId ? errors.NoSuchVersion :
- errors.NoSuchKey;
- log.trace('error no object metadata found',
- { method: 'objectPutRetention', error: err });
- return next(err, bucket);
- }
- if (objectMD.isDeleteMarker) {
- log.trace('version is a delete marker',
- { method: 'objectPutRetention' });
- return next(errors.MethodNotAllowed, bucket);
- }
- if (!bucket.isObjectLockEnabled()) {
- log.trace('object lock not enabled on bucket',
- { method: 'objectPutRetention' });
- return next(errorInstances.InvalidRequest.customizeDescription(
- 'Bucket is missing Object Lock Configuration'
- ), bucket);
- }
- return next(null, bucket, retentionInfo, objectMD);
- }),
- (bucket, retentionInfo, objectMD, next) => {
- const objLockInfo = new ObjectLockInfo({
- mode: objectMD.retentionMode,
- date: objectMD.retentionDate,
- legalHold: objectMD.legalHold,
- });
+ },
+ (retentionInfo, next) =>
+ standardMetadataValidateBucketAndObj(
+ metadataValParams,
+ request.actionImplicitDenies,
+ log,
+ (err, bucket, objectMD) => {
+ if (err) {
+ log.trace('request authorization failed', { method: 'objectPutRetention', error: err });
+ return next(err);
+ }
+ if (!objectMD) {
+ const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
+ log.trace('error no object metadata found', { method: 'objectPutRetention', error: err });
+ return next(err, bucket);
+ }
+ if (objectMD.isDeleteMarker) {
+ log.trace('version is a delete marker', { method: 'objectPutRetention' });
+ return next(errors.MethodNotAllowed, bucket);
+ }
+ if (!bucket.isObjectLockEnabled()) {
+ log.trace('object lock not enabled on bucket', { method: 'objectPutRetention' });
+ return next(
+ errorInstances.InvalidRequest.customizeDescription(
+ 'Bucket is missing Object Lock Configuration',
+ ),
+ bucket,
+ );
+ }
+ return next(null, bucket, retentionInfo, objectMD);
+ },
+ ),
+ (bucket, retentionInfo, objectMD, next) => {
+ const objLockInfo = new ObjectLockInfo({
+ mode: objectMD.retentionMode,
+ date: objectMD.retentionDate,
+ legalHold: objectMD.legalHold,
+ });
- if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) {
- return next(errors.AccessDenied, bucket);
- }
+ if (!objLockInfo.canModifyPolicy(retentionInfo, hasGovernanceBypass)) {
+ return next(errors.AccessDenied, bucket);
+ }
- return next(null, bucket, retentionInfo, objectMD);
- },
- (bucket, retentionInfo, objectMD, next) => {
- /* eslint-disable no-param-reassign */
- objectMD.retentionMode = retentionInfo.mode;
- objectMD.retentionDate = retentionInfo.date;
- const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
- const replicationInfo = getReplicationInfo(config,
- objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
- if (replicationInfo) {
- objectMD.replicationInfo = Object.assign({},
- objectMD.replicationInfo, replicationInfo);
+ return next(null, bucket, retentionInfo, objectMD);
+ },
+ (bucket, retentionInfo, objectMD, next) => {
+ /* eslint-disable no-param-reassign */
+ objectMD.retentionMode = retentionInfo.mode;
+ objectMD.retentionDate = retentionInfo.date;
+ const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
+ const replicationInfo = getReplicationInfo(
+ config,
+ objectKey,
+ bucket,
+ true,
+ 0,
+ REPLICATION_ACTION,
+ objectMD,
+ );
+ if (replicationInfo) {
+ objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo);
+ bumpMicroVersionId(objectMD);
+ }
+ objectMD.originOp = 's3:ObjectRetention:Put';
+ /* eslint-enable no-param-reassign */
+ metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err =>
+ next(err, bucket, objectMD),
+ );
+ },
+ ],
+ (err, bucket, objectMD) => {
+ const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket);
+ if (err) {
+ log.trace('error processing request', { error: err, method: 'objectPutRetention' });
+ } else {
+ pushMetric('putObjectRetention', log, {
+ authInfo,
+ bucket: bucketName,
+ keys: [objectKey],
+ versionId: objectMD ? objectMD.versionId : undefined,
+ location: objectMD ? objectMD.dataStoreName : undefined,
+ });
+ const verCfg = bucket.getVersioningConfiguration();
+ additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD);
}
- objectMD.originOp = 's3:ObjectRetention:Put';
- /* eslint-enable no-param-reassign */
- metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
- log, err => next(err, bucket, objectMD));
+ return callback(err, additionalResHeaders);
},
- ], (err, bucket, objectMD) => {
- const additionalResHeaders = collectCorsHeaders(request.headers.origin,
- request.method, bucket);
- if (err) {
- log.trace('error processing request',
- { error: err, method: 'objectPutRetention' });
- } else {
- pushMetric('putObjectRetention', log, {
- authInfo,
- bucket: bucketName,
- keys: [objectKey],
- versionId: objectMD ? objectMD.versionId : undefined,
- location: objectMD ? objectMD.dataStoreName : undefined,
- });
- const verCfg = bucket.getVersioningConfiguration();
- additionalResHeaders['x-amz-version-id'] =
- getVersionIdResHeader(verCfg, objectMD);
- }
- return callback(err, additionalResHeaders);
- });
+ );
}
module.exports = objectPutRetention;
diff --git a/lib/api/objectPutTagging.js b/lib/api/objectPutTagging.js
index ef23dcf64d..cd5326bb98 100644
--- a/lib/api/objectPutTagging.js
+++ b/lib/api/objectPutTagging.js
@@ -1,13 +1,17 @@
const async = require('async');
const { errors, s3middleware } = require('arsenal');
-const { decodeVersionId, getVersionIdResHeader, getVersionSpecificMetadataOptions } =
- require('./apiUtils/object/versioning');
+const {
+ decodeVersionId,
+ getVersionIdResHeader,
+ getVersionSpecificMetadataOptions,
+} = require('./apiUtils/object/versioning');
const { standardMetadataValidateBucketAndObj } = require('../metadata/metadataUtils');
const { pushMetric } = require('../utapi/utilities');
const monitoring = require('../utilities/monitoringHandler');
const getReplicationInfo = require('./apiUtils/object/getReplicationInfo');
+const bumpMicroVersionId = require('./apiUtils/object/bumpMicroVersionId');
const collectCorsHeaders = require('../utilities/collectCorsHeaders');
const metadata = require('../metadata/wrapper');
const { data } = require('../data/wrapper');
@@ -47,80 +51,86 @@ function objectPutTagging(authInfo, request, log, callback) {
request,
};
- return async.waterfall([
- next => standardMetadataValidateBucketAndObj(metadataValParams, request.actionImplicitDenies, log,
- (err, bucket, objectMD) => {
- if (err) {
- log.trace('request authorization failed',
- { method: 'objectPutTagging', error: err });
- return next(err);
- }
- if (!objectMD) {
- const err = reqVersionId ? errors.NoSuchVersion :
- errors.NoSuchKey;
- log.trace('error no object metadata found',
- { method: 'objectPutTagging', error: err });
- return next(err, bucket);
- }
- if (objectMD.isDeleteMarker) {
- log.trace('version is a delete marker',
- { method: 'objectPutTagging' });
- // FIXME we should return a `x-amz-delete-marker: true` header,
- // see S3C-7592
- return next(errors.MethodNotAllowed, bucket);
- }
- return next(null, bucket, objectMD);
- }),
- (bucket, objectMD, next) => {
- log.trace('parsing tag(s)');
- parseTagXml(request.post, log, (err, tags) =>
- next(err, bucket, tags, objectMD));
- },
- (bucket, tags, objectMD, next) => {
- // eslint-disable-next-line no-param-reassign
- objectMD.tags = tags;
- const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
- const replicationInfo = getReplicationInfo(config,
- objectKey, bucket, true, 0, REPLICATION_ACTION, objectMD);
- if (replicationInfo) {
+ return async.waterfall(
+ [
+ next =>
+ standardMetadataValidateBucketAndObj(
+ metadataValParams,
+ request.actionImplicitDenies,
+ log,
+ (err, bucket, objectMD) => {
+ if (err) {
+ log.trace('request authorization failed', { method: 'objectPutTagging', error: err });
+ return next(err);
+ }
+ if (!objectMD) {
+ const err = reqVersionId ? errors.NoSuchVersion : errors.NoSuchKey;
+ log.trace('error no object metadata found', { method: 'objectPutTagging', error: err });
+ return next(err, bucket);
+ }
+ if (objectMD.isDeleteMarker) {
+ log.trace('version is a delete marker', { method: 'objectPutTagging' });
+ // FIXME we should return a `x-amz-delete-marker: true` header,
+ // see S3C-7592
+ return next(errors.MethodNotAllowed, bucket);
+ }
+ return next(null, bucket, objectMD);
+ },
+ ),
+ (bucket, objectMD, next) => {
+ log.trace('parsing tag(s)');
+ parseTagXml(request.post, log, (err, tags) => next(err, bucket, tags, objectMD));
+ },
+ (bucket, tags, objectMD, next) => {
+ // eslint-disable-next-line no-param-reassign
+ objectMD.tags = tags;
+ const params = getVersionSpecificMetadataOptions(objectMD, config.nullVersionCompatMode);
+ const replicationInfo = getReplicationInfo(
+ config,
+ objectKey,
+ bucket,
+ true,
+ 0,
+ REPLICATION_ACTION,
+ objectMD,
+ );
+ if (replicationInfo) {
+ // eslint-disable-next-line no-param-reassign
+ objectMD.replicationInfo = Object.assign({}, objectMD.replicationInfo, replicationInfo);
+ bumpMicroVersionId(objectMD);
+ }
// eslint-disable-next-line no-param-reassign
- objectMD.replicationInfo = Object.assign({},
- objectMD.replicationInfo, replicationInfo);
+ objectMD.originOp = 's3:ObjectTagging:Put';
+ metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, err =>
+ next(err, bucket, objectMD),
+ );
+ },
+ (bucket, objectMD, next) =>
+ // if external backend handles tagging
+ data.objectTagging('Put', objectKey, bucket.getName(), objectMD, log, err =>
+ next(err, bucket, objectMD),
+ ),
+ ],
+ (err, bucket, objectMD) => {
+ const additionalResHeaders = collectCorsHeaders(request.headers.origin, request.method, bucket);
+ if (err) {
+ log.trace('error processing request', { error: err, method: 'objectPutTagging' });
+ monitoring.promMetrics('PUT', bucketName, err.code, 'putObjectTagging');
+ } else {
+ pushMetric('putObjectTagging', log, {
+ authInfo,
+ bucket: bucketName,
+ keys: [objectKey],
+ versionId: objectMD ? objectMD.versionId : undefined,
+ location: objectMD ? objectMD.dataStoreName : undefined,
+ });
+ monitoring.promMetrics('PUT', bucketName, '200', 'putObjectTagging');
+ const verCfg = bucket.getVersioningConfiguration();
+ additionalResHeaders['x-amz-version-id'] = getVersionIdResHeader(verCfg, objectMD);
}
- // eslint-disable-next-line no-param-reassign
- objectMD.originOp = 's3:ObjectTagging:Put';
- metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params,
- log, err =>
- next(err, bucket, objectMD));
+ return callback(err, additionalResHeaders);
},
- (bucket, objectMD, next) =>
- // if external backend handles tagging
- data.objectTagging('Put', objectKey, bucket.getName(), objectMD,
- log, err => next(err, bucket, objectMD)),
- ], (err, bucket, objectMD) => {
- const additionalResHeaders = collectCorsHeaders(request.headers.origin,
- request.method, bucket);
- if (err) {
- log.trace('error processing request', { error: err,
- method: 'objectPutTagging' });
- monitoring.promMetrics('PUT', bucketName, err.code,
- 'putObjectTagging');
- } else {
- pushMetric('putObjectTagging', log, {
- authInfo,
- bucket: bucketName,
- keys: [objectKey],
- versionId: objectMD ? objectMD.versionId : undefined,
- location: objectMD ? objectMD.dataStoreName : undefined,
- });
- monitoring.promMetrics(
- 'PUT', bucketName, '200', 'putObjectTagging');
- const verCfg = bucket.getVersioningConfiguration();
- additionalResHeaders['x-amz-version-id'] =
- getVersionIdResHeader(verCfg, objectMD);
- }
- return callback(err, additionalResHeaders);
- });
+ );
}
module.exports = objectPutTagging;
diff --git a/lib/metadata/acl.js b/lib/metadata/acl.js
index f48ab7aa42..aba5cf40ac 100644
--- a/lib/metadata/acl.js
+++ b/lib/metadata/acl.js
@@ -1,6 +1,7 @@
const { errors } = require('arsenal');
const getReplicationInfo = require('../api/apiUtils/object/getReplicationInfo');
+const bumpMicroVersionId = require('../api/apiUtils/object/bumpMicroVersionId');
const aclUtils = require('../utilities/aclUtils');
const constants = require('../../constants');
const metadata = require('../metadata/wrapper');
@@ -31,14 +32,16 @@ const acl = {
* contain the same number of elements, and all elements from one
* grant are incuded in the other grant
*/
- return oldAcl[grant].length === newAcl[grant].length
- && oldAcl[grant].every(value => newAcl[grant].includes(value));
+ return (
+ oldAcl[grant].length === newAcl[grant].length && oldAcl[grant].every(value => newAcl[grant].includes(value))
+ );
},
addObjectACL(bucket, objectKey, objectMD, addACLParams, params, log, cb) {
log.trace('updating object acl in metadata');
- const isAclUnchanged = Object.keys(objectMD.acl).length === Object.keys(addACLParams).length
- && Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams));
+ const isAclUnchanged =
+ Object.keys(objectMD.acl).length === Object.keys(addACLParams).length &&
+ Object.keys(objectMD.acl).every(grant => this._aclGrantDidNotChange(grant, objectMD.acl, addACLParams));
if (!isAclUnchanged) {
/* eslint-disable no-param-reassign */
objectMD.acl = addACLParams;
@@ -55,6 +58,7 @@ const acl = {
...objectMD.replicationInfo,
...replicationInfo,
};
+ bumpMicroVersionId(objectMD);
}
return metadata.putObjectMD(bucket.getName(), objectKey, objectMD, params, log, cb);
@@ -77,14 +81,22 @@ const acl = {
};
let validCannedACL = [];
if (resourceType === 'bucket') {
- validCannedACL =
- ['private', 'public-read', 'public-read-write',
- 'authenticated-read', 'log-delivery-write'];
+ validCannedACL = [
+ 'private',
+ 'public-read',
+ 'public-read-write',
+ 'authenticated-read',
+ 'log-delivery-write',
+ ];
} else if (resourceType === 'object') {
- validCannedACL =
- ['private', 'public-read', 'public-read-write',
- 'authenticated-read', 'bucket-owner-read',
- 'bucket-owner-full-control'];
+ validCannedACL = [
+ 'private',
+ 'public-read',
+ 'public-read-write',
+ 'authenticated-read',
+ 'bucket-owner-read',
+ 'bucket-owner-full-control',
+ ];
}
// parse canned acl
@@ -98,45 +110,34 @@ const acl = {
}
// parse grant headers
- const grantReadHeader =
- aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ');
+ const grantReadHeader = aclUtils.parseGrant(headers['x-amz-grant-read'], 'READ');
let grantWriteHeader = [];
if (resourceType === 'bucket') {
- grantWriteHeader = aclUtils
- .parseGrant(headers['x-amz-grant-write'], 'WRITE');
+ grantWriteHeader = aclUtils.parseGrant(headers['x-amz-grant-write'], 'WRITE');
}
- const grantReadACPHeader = aclUtils
- .parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP');
- const grantWriteACPHeader = aclUtils
- .parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP');
- const grantFullControlHeader = aclUtils
- .parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL');
- const allGrantHeaders =
- [].concat(grantReadHeader, grantWriteHeader,
- grantReadACPHeader, grantWriteACPHeader,
- grantFullControlHeader).filter(item => item !== undefined);
+ const grantReadACPHeader = aclUtils.parseGrant(headers['x-amz-grant-read-acp'], 'READ_ACP');
+ const grantWriteACPHeader = aclUtils.parseGrant(headers['x-amz-grant-write-acp'], 'WRITE_ACP');
+ const grantFullControlHeader = aclUtils.parseGrant(headers['x-amz-grant-full-control'], 'FULL_CONTROL');
+ const allGrantHeaders = []
+ .concat(grantReadHeader, grantWriteHeader, grantReadACPHeader, grantWriteACPHeader, grantFullControlHeader)
+ .filter(item => item !== undefined);
if (allGrantHeaders.length === 0) {
return cb(null, currentResourceACL);
}
- const usersIdentifiedByEmail = allGrantHeaders
- .filter(it => it && it.userIDType.toLowerCase() === 'emailaddress');
- const usersIdentifiedByGroup = allGrantHeaders
- .filter(item => item && item.userIDType.toLowerCase() === 'uri');
+ const usersIdentifiedByEmail = allGrantHeaders.filter(
+ it => it && it.userIDType.toLowerCase() === 'emailaddress',
+ );
+ const usersIdentifiedByGroup = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'uri');
const justEmails = usersIdentifiedByEmail.map(item => item.identifier);
- const validGroups = [
- constants.allAuthedUsersId,
- constants.publicId,
- constants.logId,
- ];
+ const validGroups = [constants.allAuthedUsersId, constants.publicId, constants.logId];
for (let i = 0; i < usersIdentifiedByGroup.length; i++) {
if (validGroups.indexOf(usersIdentifiedByGroup[i].identifier) < 0) {
return cb(errors.InvalidArgument);
}
}
- const usersIdentifiedByID = allGrantHeaders
- .filter(item => item && item.userIDType.toLowerCase() === 'id');
+ const usersIdentifiedByID = allGrantHeaders.filter(item => item && item.userIDType.toLowerCase() === 'id');
// TODO: Consider whether want to verify with Vault
// whether canonicalID is associated with existing
// account before adding to ACL
@@ -148,22 +149,22 @@ const acl = {
if (err) {
return cb(err);
}
- const reconstructedUsersIdentifiedByEmail = aclUtils.
- reconstructUsersIdentifiedByEmail(results,
- usersIdentifiedByEmail);
+ const reconstructedUsersIdentifiedByEmail = aclUtils.reconstructUsersIdentifiedByEmail(
+ results,
+ usersIdentifiedByEmail,
+ );
const allUsers = [].concat(
reconstructedUsersIdentifiedByEmail,
usersIdentifiedByGroup,
- usersIdentifiedByID);
- const revisedACL =
- aclUtils.sortHeaderGrants(allUsers, resourceACL);
+ usersIdentifiedByID,
+ );
+ const revisedACL = aclUtils.sortHeaderGrants(allUsers, resourceACL);
return cb(null, revisedACL);
});
} else {
// If don't have to look up canonicalID's just sort grants
// and add to bucket
- const revisedACL = aclUtils
- .sortHeaderGrants(allGrantHeaders, resourceACL);
+ const revisedACL = aclUtils.sortHeaderGrants(allGrantHeaders, resourceACL);
return cb(null, revisedACL);
}
return undefined;
@@ -171,4 +172,3 @@ const acl = {
};
module.exports = acl;
-
diff --git a/lib/services.js b/lib/services.js
index 97116329a4..3d515e7f8d 100644
--- a/lib/services.js
+++ b/lib/services.js
@@ -13,8 +13,7 @@ const constants = require('../constants');
const { config } = require('./Config');
const { data } = require('./data/wrapper');
const metadata = require('./metadata/wrapper');
-const { setObjectLockInformation }
- = require('./api/apiUtils/object/objectLockHelpers');
+const { setObjectLockInformation } = require('./api/apiUtils/object/objectLockHelpers');
const removeAWSChunked = require('./api/apiUtils/object/removeAWSChunked');
const { parseTagFromQuery } = s3middleware.tagging;
@@ -41,52 +40,53 @@ const services = {
// (without special increase)
// TODO: Consider implementing pagination like object listing
// with respect to bucket listing so can go beyond 10000
- metadata.listObject(bucketUsers, { prefix, maxKeys: 10000 }, log,
- (err, listResponse) => {
- // If MD responds with NoSuchBucket, this means the
- // hidden usersBucket has not yet been created for
- // the domain. If this is the case, it means
- // that no buckets in this domain have been created so
- // it follows that this particular user has no buckets.
- // So, the get service listing should not have any
- // buckets to list. By returning an empty array, the
- // getService API will just respond with the user info
- // without listing any buckets.
- if (err?.is?.NoSuchBucket) {
- log.trace('no buckets found');
- // If we checked the old user bucket, that means we
- // already checked the new user bucket. If neither the
- // old user bucket or the new user bucket exist, no buckets
- // have yet been created in the namespace so an empty
- // listing should be returned
- if (overrideUserbucket) {
- return cb(null, [], splitter);
- }
- // Since there were no results from checking the
- // new users bucket, we check the old users bucket
- return this.getService(authInfo, request, log,
- constants.oldSplitter, cb, oldUsersBucket);
- }
- if (err) {
- log.error('error from metadata', { error: err });
- return cb(err);
+ metadata.listObject(bucketUsers, { prefix, maxKeys: 10000 }, log, (err, listResponse) => {
+ // If MD responds with NoSuchBucket, this means the
+ // hidden usersBucket has not yet been created for
+ // the domain. If this is the case, it means
+ // that no buckets in this domain have been created so
+ // it follows that this particular user has no buckets.
+ // So, the get service listing should not have any
+ // buckets to list. By returning an empty array, the
+ // getService API will just respond with the user info
+ // without listing any buckets.
+ if (err?.is?.NoSuchBucket) {
+ log.trace('no buckets found');
+ // If we checked the old user bucket, that means we
+ // already checked the new user bucket. If neither the
+ // old user bucket or the new user bucket exist, no buckets
+ // have yet been created in the namespace so an empty
+ // listing should be returned
+ if (overrideUserbucket) {
+ return cb(null, [], splitter);
}
- return cb(null, listResponse.Contents, splitter);
- });
+ // Since there were no results from checking the
+ // new users bucket, we check the old users bucket
+ return this.getService(authInfo, request, log, constants.oldSplitter, cb, oldUsersBucket);
+ }
+ if (err) {
+ log.error('error from metadata', { error: err });
+ return cb(err);
+ }
+ return cb(null, listResponse.Contents, splitter);
+ });
},
- /**
- * Check that hashedStream.completedHash matches header contentMd5.
- * @param {object} contentMD5 - content-md5 header
- * @param {string} completedHash - hashed stream once completed
- * @param {RequestLogger} log - the current request logger
- * @return {boolean} - true if contentMD5 matches or is undefined,
- * false otherwise
- */
+ /**
+ * Check that hashedStream.completedHash matches header contentMd5.
+ * @param {object} contentMD5 - content-md5 header
+ * @param {string} completedHash - hashed stream once completed
+ * @param {RequestLogger} log - the current request logger
+ * @return {boolean} - true if contentMD5 matches or is undefined,
+ * false otherwise
+ */
checkHashMatchMD5(contentMD5, completedHash, log) {
if (contentMD5 && completedHash && contentMD5 !== completedHash) {
- log.debug('contentMD5 and completedHash does not match',
- { method: 'checkHashMatchMD5', completedHash, contentMD5 });
+ log.debug('contentMD5 and completedHash does not match', {
+ method: 'checkHashMatchMD5',
+ completedHash,
+ contentMD5,
+ });
return false;
}
return true;
@@ -103,15 +103,46 @@ const services = {
* @return {function} executes callback with err or ETag as arguments
*/
metadataStoreObject(bucketName, dataGetInfo, cipherBundle, params, cb) {
- const { objectKey, authInfo, size, contentMD5, checksum, metaHeaders,
- contentType, cacheControl, contentDisposition, contentEncoding,
- expires, multipart, headers, overrideMetadata, log,
- lastModifiedDate, versioning, versionId, uploadId,
- tagging, taggingCopy, replicationInfo, defaultRetention,
- dataStoreName, creationTime, retentionMode, retentionDate,
- legalHold, originOp, updateMicroVersionId, archive, oldReplayId,
- deleteNullKey, amzStorageClass, overheadField, needOplogUpdate,
- restoredEtag, bucketOwnerId } = params;
+ const {
+ objectKey,
+ authInfo,
+ size,
+ contentMD5,
+ checksum,
+ metaHeaders,
+ contentType,
+ cacheControl,
+ contentDisposition,
+ contentEncoding,
+ expires,
+ multipart,
+ headers,
+ overrideMetadata,
+ log,
+ lastModifiedDate,
+ versioning,
+ versionId,
+ uploadId,
+ tagging,
+ taggingCopy,
+ replicationInfo,
+ defaultRetention,
+ dataStoreName,
+ creationTime,
+ retentionMode,
+ retentionDate,
+ legalHold,
+ originOp,
+ updateMicroVersionId,
+ archive,
+ oldReplayId,
+ deleteNullKey,
+ amzStorageClass,
+ overheadField,
+ needOplogUpdate,
+ restoredEtag,
+ bucketOwnerId,
+ } = params;
log.trace('storing object in metadata');
assert.strictEqual(typeof bucketName, 'string');
const md = new ObjectMD();
@@ -189,19 +220,21 @@ const services = {
md.setUploadId(uploadId);
options.replayId = uploadId;
}
- // update microVersionId when overwriting metadata.
if (updateMicroVersionId) {
- md.updateMicroVersionId();
+ md.updateMicroVersionId(config.instanceId, config.replicationGroupId);
}
// update restore
if (archive) {
md.setAmzStorageClass(amzStorageClass);
- md.setArchive(new ObjectMDArchive(
- archive.archiveInfo,
- archive.restoreRequestedAt,
- archive.restoreRequestedDays,
- archive.restoreCompletedAt,
- archive.restoreWillExpireAt));
+ md.setArchive(
+ new ObjectMDArchive(
+ archive.archiveInfo,
+ archive.restoreRequestedAt,
+ archive.restoreRequestedDays,
+ archive.restoreCompletedAt,
+ archive.restoreWillExpireAt,
+ ),
+ );
md.setAmzRestore({
'ongoing-request': false,
'expiry-date': archive.restoreWillExpireAt,
@@ -287,55 +320,57 @@ const services = {
// If this is not the completion of a multipart upload or
// the creation of a delete marker, parse the headers to
// get the ACL's if any
- return async.waterfall([
- callback => {
- if (multipart || md.getIsDeleteMarker()) {
- return callback();
+ return async.waterfall(
+ [
+ callback => {
+ if (multipart || md.getIsDeleteMarker()) {
+ return callback();
+ }
+ const parseAclParams = {
+ headers,
+ resourceType: 'object',
+ acl: md.getAcl(),
+ log,
+ };
+ log.trace('parsing acl from headers');
+ acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => {
+ if (err) {
+ log.debug('error parsing acl', { error: err });
+ return callback(err);
+ }
+ md.setAcl(parsedACL);
+ return callback();
+ });
+ return null;
+ },
+ callback => metadata.putObjectMD(bucketName, objectKey, md, options, log, callback),
+ ],
+ (err, data) => {
+ if (err) {
+ log.error('error from metadata', { error: err });
+ return cb(err);
}
- const parseAclParams = {
- headers,
- resourceType: 'object',
- acl: md.getAcl(),
- log,
- };
- log.trace('parsing acl from headers');
- acl.parseAclFromHeaders(parseAclParams, (err, parsedACL) => {
- if (err) {
- log.debug('error parsing acl', { error: err });
- return callback(err);
+ log.trace('object successfully stored in metadata');
+ // if versioning is enabled, data will be returned from metadata
+ // as JSON containing a versionId which some APIs will need sent
+ // back to them
+ let versionId;
+ if (data) {
+ if (params.isNull && params.isDeleteMarker) {
+ versionId = 'null';
+ } else if (!params.isNull) {
+ versionId = JSON.parse(data).versionId;
}
- md.setAcl(parsedACL);
- return callback();
+ }
+ return cb(err, {
+ lastModified: md.getLastModified(),
+ tags: md.getTags(),
+ contentMD5,
+ versionId,
+ checksum: md.getChecksum(),
});
- return null;
},
- callback => metadata.putObjectMD(bucketName, objectKey, md,
- options, log, callback),
- ], (err, data) => {
- if (err) {
- log.error('error from metadata', { error: err });
- return cb(err);
- }
- log.trace('object successfully stored in metadata');
- // if versioning is enabled, data will be returned from metadata
- // as JSON containing a versionId which some APIs will need sent
- // back to them
- let versionId;
- if (data) {
- if (params.isNull && params.isDeleteMarker) {
- versionId = 'null';
- } else if (!params.isNull) {
- versionId = JSON.parse(data).versionId;
- }
- }
- return cb(err, {
- lastModified: md.getLastModified(),
- tags: md.getTags(),
- contentMD5,
- versionId,
- checksum: md.getChecksum(),
- });
- });
+ );
},
/**
@@ -358,7 +393,11 @@ const services = {
assert.strictEqual(typeof objectMD, 'object');
function deleteMDandData() {
- return metadata.deleteObjectMD(bucketName, objectKey, options, log,
+ return metadata.deleteObjectMD(
+ bucketName,
+ objectKey,
+ options,
+ log,
(err, res) => {
if (err) {
return cb(err, res);
@@ -369,8 +408,7 @@ const services = {
}
if (deferLocationDeletion) {
- return cb(null, Array.isArray(objectMD.location)
- ? objectMD.location : [objectMD.location]);
+ return cb(null, Array.isArray(objectMD.location) ? objectMD.location : [objectMD.location]);
}
if (!Array.isArray(objectMD.location)) {
@@ -384,14 +422,15 @@ const services = {
}
return cb(null, res);
});
- }, originOp);
+ },
+ originOp,
+ );
}
const objGetInfo = objectMD.location;
// special case that prevents azure blocks from unecessary deletion
// will return null if no need
- return data.protectAzureBlocks(bucketName, objectKey, objGetInfo,
- log, err => {
+ return data.protectAzureBlocks(bucketName, objectKey, objGetInfo, log, err => {
if (err) {
return cb(err);
}
@@ -411,16 +450,14 @@ const services = {
*/
getObjectListing(bucketName, listingParams, log, cb) {
assert.strictEqual(typeof bucketName, 'string');
- log.trace('performing metadata get object listing',
- { listingParams });
- metadata.listObject(bucketName, listingParams, log,
- (err, listResponse) => {
- if (err) {
- log.debug('error from metadata', { error: err });
- return cb(err);
- }
- return cb(null, listResponse);
- });
+ log.trace('performing metadata get object listing', { listingParams });
+ metadata.listObject(bucketName, listingParams, log, (err, listResponse) => {
+ if (err) {
+ log.debug('error from metadata', { error: err });
+ return cb(err);
+ }
+ return cb(null, listResponse);
+ });
},
/**
@@ -463,10 +500,8 @@ const services = {
}
// Check each version in current batch for matching uploadId
- const matchedVersion = (listResponse.Versions || []).find(version =>
- version.key === objectKey &&
- version.value &&
- version.value.uploadId === uploadId
+ const matchedVersion = (listResponse.Versions || []).find(
+ version => version.key === objectKey && version.value && version.value.uploadId === uploadId,
);
if (matchedVersion) {
@@ -484,7 +519,7 @@ const services = {
return callback();
});
},
- err => cb(err, err ? null : foundVersion)
+ err => cb(err, err ? null : foundVersion),
);
},
@@ -500,16 +535,14 @@ const services = {
*/
getLifecycleListing(bucketName, listingParams, log, cb) {
assert.strictEqual(typeof bucketName, 'string');
- log.trace('performing metadata get object listing for lifecycle',
- { listingParams });
- metadata.listLifecycleObject(bucketName, listingParams, log,
- (err, listResponse) => {
- if (err) {
- log.debug('error from metadata', { error: err });
- return cb(err);
- }
- return cb(null, listResponse);
- });
+ log.trace('performing metadata get object listing for lifecycle', { listingParams });
+ metadata.listLifecycleObject(bucketName, listingParams, log, (err, listResponse) => {
+ if (err) {
+ log.debug('error from metadata', { error: err });
+ return cb(err);
+ }
+ return cb(null, listResponse);
+ });
},
metadataStoreMPObject(bucketName, cipherBundle, params, log, cb) {
@@ -522,9 +555,7 @@ const services = {
// the splitter.
// 2) UploadId's are UUID version 4
const splitter = params.splitter;
- const longMPUIdentifier =
- `overview${splitter}${params.objectKey}` +
- `${splitter}${params.uploadId}`;
+ const longMPUIdentifier = `overview${splitter}${params.objectKey}` + `${splitter}${params.uploadId}`;
const multipartObjectMD = {};
multipartObjectMD.id = params.uploadId;
multipartObjectMD.eventualStorageBucket = params.eventualStorageBucket;
@@ -546,28 +577,19 @@ const services = {
multipartObjectMD.key = params.objectKey;
multipartObjectMD.uploadId = params.uploadId;
multipartObjectMD['cache-control'] = params.headers['cache-control'];
- multipartObjectMD['content-disposition'] =
- params.headers['content-disposition'];
- multipartObjectMD['content-encoding'] =
- removeAWSChunked(params.headers['content-encoding']);
- multipartObjectMD['content-type'] =
- params.headers['content-type'];
- multipartObjectMD.expires =
- params.headers.expires;
- multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639
- multipartObjectMD['x-amz-website-redirect-location'] =
- params.headers['x-amz-website-redirect-location'];
+ multipartObjectMD['content-disposition'] = params.headers['content-disposition'];
+ multipartObjectMD['content-encoding'] = removeAWSChunked(params.headers['content-encoding']);
+ multipartObjectMD['content-type'] = params.headers['content-type'];
+ multipartObjectMD.expires = params.headers.expires;
+ multipartObjectMD['x-amz-storage-class'] = params.storageClass; // TODO: removed CLDSRV-639
+ multipartObjectMD['x-amz-website-redirect-location'] = params.headers['x-amz-website-redirect-location'];
if (cipherBundle) {
- multipartObjectMD['x-amz-server-side-encryption'] =
- cipherBundle.algorithm;
+ multipartObjectMD['x-amz-server-side-encryption'] = cipherBundle.algorithm;
if (cipherBundle.masterKeyId) {
- multipartObjectMD[
- 'x-amz-server-side-encryption-aws-kms-key-id'] =
- cipherBundle.masterKeyId;
+ multipartObjectMD['x-amz-server-side-encryption-aws-kms-key-id'] = cipherBundle.masterKeyId;
}
}
- multipartObjectMD.controllingLocationConstraint =
- params.controllingLocationConstraint;
+ multipartObjectMD.controllingLocationConstraint = params.controllingLocationConstraint;
multipartObjectMD.dataStoreName = params.dataStoreName;
if (params.tagging) {
const validationTagRes = parseTagFromQuery(params.tagging);
@@ -613,15 +635,14 @@ const services = {
return cb(err);
}
multipartObjectMD.acl = parsedACL;
- metadata.putObjectMD(bucketName, longMPUIdentifier,
- multipartObjectMD, {}, log, err => {
- if (err) {
- log.error('error from metadata', { error: err });
- return cb(err);
- }
+ metadata.putObjectMD(bucketName, longMPUIdentifier, multipartObjectMD, {}, log, err => {
+ if (err) {
+ log.error('error from metadata', { error: err });
+ return cb(err);
+ }
- return cb(null, multipartObjectMD);
- });
+ return cb(null, multipartObjectMD);
+ });
return undefined;
});
},
@@ -648,12 +669,10 @@ const services = {
assert.strictEqual(typeof params.splitter, 'string');
assert.strictEqual(typeof params.storedMetadata, 'object');
const splitter = params.splitter;
- const longMPUIdentifier =
- `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`;
+ const longMPUIdentifier = `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`;
const multipartObjectMD = Object.assign({}, params.storedMetadata);
multipartObjectMD.completeInProgress = true;
- metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD,
- {}, log, err => {
+ metadata.putObjectMD(params.bucketName, longMPUIdentifier, multipartObjectMD, {}, log, err => {
if (err) {
log.error('error from metadata', { error: err });
return cb(err);
@@ -685,30 +704,28 @@ const services = {
const mpuBucketName = `${constants.mpuBucketPrefix}${params.bucketName}`;
const splitter = params.splitter;
- const mpuOverviewKey =
- `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`;
- return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log,
- (err, res) => {
- if (err) {
- if (err.is && err.is.NoSuchKey) {
- // The overview key no longer exists, meaning completeMultipartUpload
- // already ran to completion and cleaned up the MPU bucket.
- // This is a race condition: objectPutPart checked for old
- // part locations after completeMultipartUpload deleted the overview.
- // Returning true (complete in progress) prevents objectPutPart
- // from deleting part data that may have already been committed
- // as the final object.
- return cb(null, true);
- }
- log.error('error getting the overview object from mpu bucket', {
- error: err,
- method: 'services.isCompleteMPUInProgress',
- params,
- });
- return cb(err);
+ const mpuOverviewKey = `overview${splitter}${params.objectKey}${splitter}${params.uploadId}`;
+ return metadata.getObjectMD(mpuBucketName, mpuOverviewKey, {}, log, (err, res) => {
+ if (err) {
+ if (err.is && err.is.NoSuchKey) {
+ // The overview key no longer exists, meaning completeMultipartUpload
+ // already ran to completion and cleaned up the MPU bucket.
+ // This is a race condition: objectPutPart checked for old
+ // part locations after completeMultipartUpload deleted the overview.
+ // Returning true (complete in progress) prevents objectPutPart
+ // from deleting part data that may have already been committed
+ // as the final object.
+ return cb(null, true);
}
- return cb(null, Boolean(res.completeInProgress));
- });
+ log.error('error getting the overview object from mpu bucket', {
+ error: err,
+ method: 'services.isCompleteMPUInProgress',
+ params,
+ });
+ return cb(err);
+ }
+ return cb(null, Boolean(res.completeInProgress));
+ });
},
/**
@@ -725,8 +742,7 @@ const services = {
* - the overview key stored metadata
*/
metadataValidateMultipart(params, cb) {
- const { bucketName, uploadId, authInfo,
- objectKey, requestType, log } = params;
+ const { bucketName, uploadId, authInfo, objectKey, requestType, log } = params;
assert.strictEqual(typeof bucketName, 'string');
// This checks whether the mpu bucket exists.
@@ -734,13 +750,11 @@ const services = {
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
metadata.getBucket(mpuBucketName, log, (err, mpuBucket) => {
if (err?.is?.NoSuchBucket) {
- log.debug('bucket not found in metadata', { error: err,
- method: 'services.metadataValidateMultipart' });
+ log.debug('bucket not found in metadata', { error: err, method: 'services.metadataValidateMultipart' });
return cb(errors.NoSuchUpload);
}
if (err) {
- log.error('error from metadata', { error: err,
- method: 'services.metadataValidateMultipart' });
+ log.error('error from metadata', { error: err, method: 'services.metadataValidateMultipart' });
return cb(err);
}
@@ -749,87 +763,78 @@ const services = {
if (mpuBucket.getMdBucketModelVersion() < 2) {
splitter = constants.oldSplitter;
}
- const mpuOverviewKey =
- `overview${splitter}${objectKey}${splitter}${uploadId}`;
+ const mpuOverviewKey = `overview${splitter}${objectKey}${splitter}${uploadId}`;
- metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey,
- {}, log, (err, storedMetadata) => {
- if (err) {
- if (err.is && err.is.NoSuchKey) {
- return cb(errors.NoSuchUpload);
- }
- log.error('error from metadata', { error: err });
- return cb(err);
+ metadata.getObjectMD(mpuBucket.getName(), mpuOverviewKey, {}, log, (err, storedMetadata) => {
+ if (err) {
+ if (err.is && err.is.NoSuchKey) {
+ return cb(errors.NoSuchUpload);
}
+ log.error('error from metadata', { error: err });
+ return cb(err);
+ }
- const initiatorID = storedMetadata.initiator.ID;
- const ownerID = storedMetadata['owner-id'];
- const mpuOverview = {
- key: storedMetadata.key,
- id: storedMetadata.id,
- eventualStorageBucket:
- storedMetadata.eventualStorageBucket,
- initiatorID,
- initiatorDisplayName:
- storedMetadata.initiator.DisplayName,
- ownerID,
- ownerDisplayName:
- storedMetadata['owner-display-name'],
- storageClass:
- storedMetadata['x-amz-storage-class'],
- initiated: storedMetadata.initiated,
- controllingLocationConstraint:
- storedMetadata.controllingLocationConstraint,
- checksumAlgorithm: storedMetadata.checksumAlgorithm,
- checksumType: storedMetadata.checksumType,
- checksumIsDefault: storedMetadata.checksumIsDefault,
- };
+ const initiatorID = storedMetadata.initiator.ID;
+ const ownerID = storedMetadata['owner-id'];
+ const mpuOverview = {
+ key: storedMetadata.key,
+ id: storedMetadata.id,
+ eventualStorageBucket: storedMetadata.eventualStorageBucket,
+ initiatorID,
+ initiatorDisplayName: storedMetadata.initiator.DisplayName,
+ ownerID,
+ ownerDisplayName: storedMetadata['owner-display-name'],
+ storageClass: storedMetadata['x-amz-storage-class'],
+ initiated: storedMetadata.initiated,
+ controllingLocationConstraint: storedMetadata.controllingLocationConstraint,
+ checksumAlgorithm: storedMetadata.checksumAlgorithm,
+ checksumType: storedMetadata.checksumType,
+ checksumIsDefault: storedMetadata.checksumIsDefault,
+ };
- const tagging = storedMetadata['x-amz-tagging'];
- if (tagging) {
- mpuOverview.tagging = tagging;
- }
- // If access was provided by the destination bucket's
- // bucket policies, go ahead.
- if (requestType === 'bucketPolicyGoAhead') {
- return cb(null, mpuBucket, mpuOverview, storedMetadata);
- }
+ const tagging = storedMetadata['x-amz-tagging'];
+ if (tagging) {
+ mpuOverview.tagging = tagging;
+ }
+ // If access was provided by the destination bucket's
+ // bucket policies, go ahead.
+ if (requestType === 'bucketPolicyGoAhead') {
+ return cb(null, mpuBucket, mpuOverview, storedMetadata);
+ }
- const requesterID = authInfo.isRequesterAnIAMUser() ?
- authInfo.getArn() : authInfo.getCanonicalID();
- const isRequesterInitiator =
- initiatorID === requesterID;
- const isRequesterParentAccountOfInitiator =
- ownerID === authInfo.getCanonicalID();
- if (requestType === 'putPart or complete') {
- // Only the initiator of the multipart
- // upload can upload a part or complete the mpu
- if (!isRequesterInitiator) {
- return cb(errors.AccessDenied);
- }
+ const requesterID = authInfo.isRequesterAnIAMUser() ? authInfo.getArn() : authInfo.getCanonicalID();
+ const isRequesterInitiator = initiatorID === requesterID;
+ const isRequesterParentAccountOfInitiator = ownerID === authInfo.getCanonicalID();
+ if (requestType === 'putPart or complete') {
+ // Only the initiator of the multipart
+ // upload can upload a part or complete the mpu
+ if (!isRequesterInitiator) {
+ return cb(errors.AccessDenied);
}
- if (requestType === 'deleteMPU'
- || requestType === 'listParts') {
- // In order for account/user to be
- // authorized must either be the
- // bucket owner or intitator of
- // the multipart upload request
- // (or parent account of initiator).
- // In addition if the bucket policy
- // designates someone else with
- // s3:AbortMultipartUpload or
- // s3:ListMultipartUploadPartsrights,
- // as applicable, that account/user will have the right.
- // If got to this step, it means there is
- // no bucket policy on this.
- if (mpuBucket.getOwner() !== authInfo.getCanonicalID()
- && !isRequesterInitiator
- && !isRequesterParentAccountOfInitiator) {
- return cb(errors.AccessDenied);
- }
+ }
+ if (requestType === 'deleteMPU' || requestType === 'listParts') {
+ // In order for account/user to be
+ // authorized must either be the
+ // bucket owner or intitator of
+ // the multipart upload request
+ // (or parent account of initiator).
+ // In addition if the bucket policy
+ // designates someone else with
+ // s3:AbortMultipartUpload or
+ // s3:ListMultipartUploadPartsrights,
+ // as applicable, that account/user will have the right.
+ // If got to this step, it means there is
+ // no bucket policy on this.
+ if (
+ mpuBucket.getOwner() !== authInfo.getCanonicalID() &&
+ !isRequesterInitiator &&
+ !isRequesterParentAccountOfInitiator
+ ) {
+ return cb(errors.AccessDenied);
}
- return cb(null, mpuBucket, mpuOverview, storedMetadata);
- });
+ }
+ return cb(null, mpuBucket, mpuOverview, storedMetadata);
+ });
return undefined;
});
},
@@ -851,13 +856,11 @@ const services = {
* @param {function} cb - callback to send error or move to next task
* @return {undefined}
*/
- metadataStorePart(mpuBucketName, partLocations,
- metaStoreParams, log, cb) {
+ metadataStorePart(mpuBucketName, partLocations, metaStoreParams, log, cb) {
assert.strictEqual(typeof mpuBucketName, 'string');
- const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId }
- = metaStoreParams;
- const dateModified = typeof lastModified === 'string' ?
- lastModified : new Date().toJSON();
+ const { partNumber, contentMD5, size, uploadId, lastModified, splitter, overheadField, ownerId } =
+ metaStoreParams;
+ const dateModified = typeof lastModified === 'string' ? lastModified : new Date().toJSON();
assert.strictEqual(typeof splitter, 'string');
const partKey = `${uploadId}${splitter}${partNumber}`;
const omVal = {
@@ -865,7 +868,7 @@ const services = {
// from an object to an array
'md-model-version': 3,
partLocations,
- 'key': partKey,
+ key: partKey,
'last-modified': dateModified,
'content-md5': contentMD5,
'content-length': size,
@@ -887,14 +890,14 @@ const services = {
},
/**
- * Gets list of open multipart uploads in bucket
- * @param {object} MPUbucketName - bucket in which objectMetadata is stored
- * @param {object} listingParams - params object passing on
- * needed items from request object
- * @param {object} log - Werelogs logger
- * @param {function} cb - callback to listMultipartUploads.js
- * @return {undefined}
- */
+ * Gets list of open multipart uploads in bucket
+ * @param {object} MPUbucketName - bucket in which objectMetadata is stored
+ * @param {object} listingParams - params object passing on
+ * needed items from request object
+ * @param {object} log - Werelogs logger
+ * @param {function} cb - callback to listMultipartUploads.js
+ * @return {undefined}
+ */
getMultipartUploadListing(MPUbucketName, listingParams, log, cb) {
assert.strictEqual(typeof MPUbucketName, 'string');
assert.strictEqual(typeof listingParams.splitter, 'string');
@@ -921,8 +924,7 @@ const services = {
if (bucket.getMdBucketModelVersion() < 2) {
listParams.splitter = constants.oldSplitter;
}
- metadata.listMultipartUploads(MPUbucketName, listParams, log,
- cb);
+ metadata.listMultipartUploads(MPUbucketName, listParams, log, cb);
return undefined;
});
},
@@ -944,24 +946,26 @@ const services = {
if (err?.is?.NoSuchBucket) {
log.trace('no buckets found');
const creationDate = new Date().toJSON();
- const mpuBucket = new BucketInfo(MPUBucketName,
+ const mpuBucket = new BucketInfo(
+ MPUBucketName,
destinationBucket.getOwner(),
- destinationBucket.getOwnerDisplayName(), creationDate,
- BucketInfo.currentModelVersion());
+ destinationBucket.getOwnerDisplayName(),
+ creationDate,
+ BucketInfo.currentModelVersion(),
+ );
// Note that unlike during the creation of a normal bucket,
// we do NOT add this bucket to the lists of a user's buckets.
// By not adding this bucket to the lists of a user's buckets,
// a getService request should not return a reference to this
// bucket. This is the desired behavior since this should be
// a hidden bucket.
- return metadata.createBucket(MPUBucketName, mpuBucket, log,
- err => {
- if (err) {
- log.error('error from metadata', { error: err });
- return cb(err);
- }
- return cb(null, mpuBucket);
- });
+ return metadata.createBucket(MPUBucketName, mpuBucket, log, err => {
+ if (err) {
+ log.error('error from metadata', { error: err });
+ return cb(err);
+ }
+ return cb(null, mpuBucket);
+ });
}
if (err) {
log.error('error from metadata', {
@@ -986,8 +990,7 @@ const services = {
},
getSomeMPUparts(params, cb) {
- const { uploadId, mpuBucketName, maxParts, partNumberMarker, log } =
- params;
+ const { uploadId, mpuBucketName, maxParts, partNumberMarker, log } = params;
assert.strictEqual(typeof mpuBucketName, 'string');
assert.strictEqual(typeof params.splitter, 'string');
const paddedPartNumber = `000000${partNumberMarker}`.substr(-5);
@@ -1004,9 +1007,14 @@ const services = {
// If have efficient way to batch delete metadata, should so this
// all at once in production implementation
assert.strictEqual(typeof mpuBucketName, 'string');
- async.eachLimit(keysToDelete, 5, (key, callback) => {
- metadata.deleteObjectMD(mpuBucketName, key, { overheadField: constants.overheadField }, log, callback);
- }, cb);
+ async.eachLimit(
+ keysToDelete,
+ 5,
+ (key, callback) => {
+ metadata.deleteObjectMD(mpuBucketName, key, { overheadField: constants.overheadField }, log, callback);
+ },
+ cb,
+ );
},
};
diff --git a/lib/utilities/collectResponseHeaders.js b/lib/utilities/collectResponseHeaders.js
index a754300c62..8480cf8d9f 100644
--- a/lib/utilities/collectResponseHeaders.js
+++ b/lib/utilities/collectResponseHeaders.js
@@ -1,6 +1,5 @@
const { getVersionIdResHeader } = require('../api/apiUtils/object/versioning');
-const checkUserMetadataSize
- = require('../api/apiUtils/object/checkUserMetadataSize');
+const checkUserMetadataSize = require('../api/apiUtils/object/checkUserMetadataSize');
const { getAmzRestoreResHeader } = require('../api/apiUtils/object/coldStorage');
const { config } = require('../Config');
const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface');
@@ -16,38 +15,36 @@ const { getKeyIdFromArn } = require('arsenal/build/lib/network/KMSInterface');
* @return {object} responseMetaHeaders headers with object metadata to include
* in response to client
*/
-function collectResponseHeaders(objectMD, corsHeaders, versioningCfg,
- returnTagCount) {
+function collectResponseHeaders(objectMD, corsHeaders, versioningCfg, returnTagCount) {
// Add user meta headers from objectMD
let responseMetaHeaders = Object.assign({}, corsHeaders);
- Object.keys(objectMD).filter(val => (val.startsWith('x-amz-meta-')))
- .forEach(id => { responseMetaHeaders[id] = objectMD[id]; });
+ Object.keys(objectMD)
+ .filter(val => val.startsWith('x-amz-meta-'))
+ .forEach(id => {
+ responseMetaHeaders[id] = objectMD[id];
+ });
// Check user metadata size
responseMetaHeaders = checkUserMetadataSize(responseMetaHeaders);
// TODO: When implement lifecycle, add additional response headers
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
- responseMetaHeaders['x-amz-version-id'] =
- getVersionIdResHeader(versioningCfg, objectMD);
+ responseMetaHeaders['x-amz-version-id'] = getVersionIdResHeader(versioningCfg, objectMD);
if (objectMD['x-amz-website-redirect-location']) {
- responseMetaHeaders['x-amz-website-redirect-location'] =
- objectMD['x-amz-website-redirect-location'];
+ responseMetaHeaders['x-amz-website-redirect-location'] = objectMD['x-amz-website-redirect-location'];
}
if (objectMD['x-amz-storage-class'] !== 'STANDARD') {
- responseMetaHeaders['x-amz-storage-class'] =
- objectMD['x-amz-storage-class'];
+ responseMetaHeaders['x-amz-storage-class'] = objectMD['x-amz-storage-class'];
}
if (objectMD['x-amz-server-side-encryption']) {
- responseMetaHeaders['x-amz-server-side-encryption']
- = objectMD['x-amz-server-side-encryption'];
+ responseMetaHeaders['x-amz-server-side-encryption'] = objectMD['x-amz-server-side-encryption'];
}
const kmsKey = objectMD['x-amz-server-side-encryption-aws-kms-key-id'];
- if (kmsKey &&
- objectMD['x-amz-server-side-encryption'] === 'aws:kms') {
- responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id']
- = config.kmsHideScalityArn ? getKeyIdFromArn(kmsKey) : kmsKey;
+ if (kmsKey && objectMD['x-amz-server-side-encryption'] === 'aws:kms') {
+ responseMetaHeaders['x-amz-server-side-encryption-aws-kms-key-id'] = config.kmsHideScalityArn
+ ? getKeyIdFromArn(kmsKey)
+ : kmsKey;
}
const restoreHeader = getAmzRestoreResHeader(objectMD);
@@ -65,8 +62,7 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg,
responseMetaHeaders['Cache-Control'] = objectMD['cache-control'];
}
if (objectMD['content-disposition']) {
- responseMetaHeaders['Content-Disposition']
- = objectMD['content-disposition'];
+ responseMetaHeaders['Content-Disposition'] = objectMD['content-disposition'];
}
if (objectMD['content-encoding']) {
responseMetaHeaders['Content-Encoding'] = objectMD['content-encoding'];
@@ -78,40 +74,33 @@ function collectResponseHeaders(objectMD, corsHeaders, versioningCfg,
// Note: ETag must have a capital "E" and capital "T" for cosbench
// to work.
responseMetaHeaders.ETag = `"${objectMD['content-md5']}"`;
- responseMetaHeaders['Last-Modified'] =
- new Date(objectMD['last-modified']).toUTCString();
+ responseMetaHeaders['Last-Modified'] = new Date(objectMD['last-modified']).toUTCString();
if (objectMD['content-type']) {
responseMetaHeaders['Content-Type'] = objectMD['content-type'];
}
- if (returnTagCount && objectMD.tags &&
- Object.keys(objectMD.tags).length > 0) {
- responseMetaHeaders['x-amz-tagging-count'] =
- Object.keys(objectMD.tags).length;
+ if (returnTagCount && objectMD.tags && Object.keys(objectMD.tags).length > 0) {
+ responseMetaHeaders['x-amz-tagging-count'] = Object.keys(objectMD.tags).length;
}
- const hasRetentionInfo = objectMD.retentionMode
- && objectMD.retentionDate;
+ const hasRetentionInfo = objectMD.retentionMode && objectMD.retentionDate;
if (hasRetentionInfo) {
- responseMetaHeaders['x-amz-object-lock-retain-until-date']
- = objectMD.retentionDate;
- responseMetaHeaders['x-amz-object-lock-mode']
- = objectMD.retentionMode;
+ responseMetaHeaders['x-amz-object-lock-retain-until-date'] = objectMD.retentionDate;
+ responseMetaHeaders['x-amz-object-lock-mode'] = objectMD.retentionMode;
}
if (objectMD.legalHold !== undefined) {
- responseMetaHeaders['x-amz-object-lock-legal-hold']
- = objectMD.legalHold ? 'ON' : 'OFF';
+ responseMetaHeaders['x-amz-object-lock-legal-hold'] = objectMD.legalHold ? 'ON' : 'OFF';
}
- if (objectMD.replicationInfo && objectMD.replicationInfo.status) {
- responseMetaHeaders['x-amz-replication-status'] =
- objectMD.replicationInfo.status;
+ if (objectMD.replicationInfo) {
+ const { isReplica, status } = objectMD.replicationInfo;
+ if (isReplica || status) {
+ responseMetaHeaders['x-amz-replication-status'] = isReplica ? 'REPLICA' : status;
+ }
}
if (Array.isArray(objectMD?.replicationInfo?.backends)) {
objectMD.replicationInfo.backends.forEach(backend => {
const { status, site, dataStoreVersionId } = backend;
- responseMetaHeaders[`x-amz-meta-${site}-replication-status`] =
- status;
+ responseMetaHeaders[`x-amz-meta-${site}-replication-status`] = status;
if (status === 'COMPLETED' && dataStoreVersionId) {
- responseMetaHeaders[`x-amz-meta-${site}-version-id`] =
- dataStoreVersionId;
+ responseMetaHeaders[`x-amz-meta-${site}-version-id`] = dataStoreVersionId;
}
});
}
diff --git a/package.json b/package.json
index 999993962b..235207a6f0 100644
--- a/package.json
+++ b/package.json
@@ -33,7 +33,7 @@
"@azure/storage-blob": "^12.28.0",
"@hapi/joi": "^17.1.1",
"@smithy/node-http-handler": "^3.0.0",
- "arsenal": "git+https://github.com/scality/Arsenal#8.4.4",
+ "arsenal": "git+https://github.com/scality/Arsenal#2c429ab35a5ac82c3dafa5a0296a49a23a9c8a4a",
"async": "2.6.4",
"bucketclient": "scality/bucketclient#8.2.7",
"bufferutil": "^4.0.8",
diff --git a/tests/unit/api/apiUtils/getReplicationInfo.js b/tests/unit/api/apiUtils/getReplicationInfo.js
index d8bec4c1e4..d61ecbf6c8 100644
--- a/tests/unit/api/apiUtils/getReplicationInfo.js
+++ b/tests/unit/api/apiUtils/getReplicationInfo.js
@@ -2,15 +2,25 @@ const assert = require('assert');
const BucketInfo = require('arsenal').models.BucketInfo;
const AuthInfo = require('arsenal').auth.AuthInfo;
-const getReplicationInfo =
- require('../../../../lib/api/apiUtils/object/getReplicationInfo');
+const getReplicationInfo = require('../../../../lib/api/apiUtils/object/getReplicationInfo');
function _getObjectReplicationInfo(s3config, replicationConfig) {
const bucketInfo = new BucketInfo(
- 'testbucket', 'someCanonicalId', 'accountDisplayName',
+ 'testbucket',
+ 'someCanonicalId',
+ 'accountDisplayName',
new Date().toJSON(),
- null, null, null, null, null, null, null, null, null,
- replicationConfig);
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ replicationConfig,
+ );
return getReplicationInfo(s3config, 'fookey', bucketInfo, true, 123, null, null);
}
@@ -36,56 +46,66 @@ const TEST_CONFIG = {
azureStorageAccountName: 'fakeaccountname',
azureStorageAccessKey: 'Fake00Key001',
bucketMatch: true,
- azureContainerName: 's3test'
- }
+ azureContainerName: 's3test',
+ },
},
},
- replicationEndpoints: [{
- site: 'zenko',
- servers: ['127.0.0.1:8000'],
- default: true,
- }, {
- site: 'us-east-2',
- type: 'aws_s3',
- }],
+ replicationEndpoints: [
+ {
+ site: 'zenko',
+ servers: ['127.0.0.1:8000'],
+ default: true,
+ },
+ {
+ site: 'us-east-2',
+ type: 'aws_s3',
+ },
+ ],
};
describe('getReplicationInfo helper', () => {
it('should get replication info when rules are enabled', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend',
+ },
+ ],
destination: 'tosomewhere',
};
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'awsbackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'awsbackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3',
isNFS: undefined,
+ isReplica: false,
});
});
it('should not get replication info when rules are disabled', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: false,
- storageClass: 'awsbackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: false,
+ storageClass: 'awsbackend',
+ },
+ ],
destination: 'tosomewhere',
};
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
@@ -95,231 +115,281 @@ describe('getReplicationInfo helper', () => {
it('should get replication info with single cloud target', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend',
+ },
+ ],
destination: 'tosomewhere',
};
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'awsbackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'awsbackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3',
isNFS: undefined,
+ isReplica: false,
});
});
it('should get replication info with multiple cloud targets', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend,azurebackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend,azurebackend',
+ },
+ ],
destination: 'tosomewhere',
};
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'awsbackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }, {
- site: 'azurebackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'awsbackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ {
+ site: 'azurebackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend,azurebackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3,azure',
isNFS: undefined,
+ isReplica: false,
});
});
- it('should get replication info with multiple cloud targets and ' +
- 'preferred read location', () => {
+ it('should get replication info with multiple cloud targets and ' + 'preferred read location', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend:preferred_read,azurebackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend:preferred_read,azurebackend',
+ },
+ ],
destination: 'tosomewhere',
preferredReadLocation: 'awsbackend',
};
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'awsbackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }, {
- site: 'azurebackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'awsbackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ {
+ site: 'azurebackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend:preferred_read,azurebackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3,azure',
isNFS: undefined,
+ isReplica: false,
});
});
- it('should not get replication info when service account type ' +
- 'cannot trigger replication', () => {
+ it('should not get replication info when service account type ' + 'cannot trigger replication', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend',
+ },
+ ],
destination: 'tosomewhere',
};
const bucketInfo = new BucketInfo(
- 'testbucket', 'abcdef/lifecycle', 'Lifecycle Service Account',
+ 'testbucket',
+ 'abcdef/lifecycle',
+ 'Lifecycle Service Account',
new Date().toJSON(),
- null, null, null, null, null, null, null, null, null,
- replicationConfig);
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ replicationConfig,
+ );
const authInfo = new AuthInfo({
canonicalID: 'abcdef/lifecycle',
accountDisplayName: 'Lifecycle Service Account',
});
- const replicationInfo = getReplicationInfo(TEST_CONFIG,
- 'fookey', bucketInfo, true, 123, null, null, authInfo);
+ const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo);
assert.deepStrictEqual(replicationInfo, undefined);
});
- it('should get replication info when service account type can ' +
- 'trigger replication', () => {
+ it('should get replication info when service account type can ' + 'trigger replication', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend',
+ },
+ ],
destination: 'tosomewhere',
};
const bucketInfo = new BucketInfo(
- 'testbucket', 'abcdef/md-ingestion',
+ 'testbucket',
+ 'abcdef/md-ingestion',
'Metadata Ingestion Service Account',
new Date().toJSON(),
- null, null, null, null, null, null, null, null, null,
- replicationConfig);
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ replicationConfig,
+ );
const authInfo = new AuthInfo({
canonicalID: 'abcdef/md-ingestion',
accountDisplayName: 'Metadata Ingestion Service Account',
});
- const replicationInfo = getReplicationInfo(TEST_CONFIG,
- 'fookey', bucketInfo, true, 123, null, null, authInfo);
+ const replicationInfo = getReplicationInfo(TEST_CONFIG, 'fookey', bucketInfo, true, 123, null, null, authInfo);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'awsbackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'awsbackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3',
isNFS: undefined,
+ isReplica: false,
});
});
it('should get replication info with default StorageClass when rules are enabled', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2',
- rules: [{
- prefix: '',
- enabled: true,
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ },
+ ],
destination: 'tosomewhere',
};
const replicationInfo = _getObjectReplicationInfo(TEST_CONFIG, replicationConfig);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'zenko',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'zenko',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'zenko',
role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2',
storageType: '',
isNFS: undefined,
+ isReplica: false,
});
});
it('should return undefined with specified StorageClass mode if no replication endpoint is configured', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role',
- rules: [{
- prefix: '',
- enabled: true,
- storageClass: 'awsbackend',
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ storageClass: 'awsbackend',
+ },
+ ],
destination: 'tosomewhere',
};
const configWithNoReplicationEndpoint = {
locationConstraints: TEST_CONFIG.locationConstraints,
replicationEndpoints: [],
};
- const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint,
- replicationConfig);
+ const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig);
assert.deepStrictEqual(replicationInfo, {
status: 'PENDING',
- backends: [{
- site: 'awsbackend',
- status: 'PENDING',
- dataStoreVersionId: '',
- }],
+ backends: [
+ {
+ site: 'awsbackend',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
content: ['METADATA'],
destination: 'tosomewhere',
storageClass: 'awsbackend',
role: 'arn:aws:iam::root:role/s3-replication-role',
storageType: 'aws_s3',
isNFS: undefined,
+ isReplica: false,
});
});
it('should return undefined with default StorageClass if no replication endpoint is configured', () => {
const replicationConfig = {
role: 'arn:aws:iam::root:role/s3-replication-role-1,arn:aws:iam::root:role/s3-replication-role-2',
- rules: [{
- prefix: '',
- enabled: true,
- }],
+ rules: [
+ {
+ prefix: '',
+ enabled: true,
+ },
+ ],
destination: 'tosomewhere',
};
const configWithNoReplicationEndpoint = {
locationConstraints: TEST_CONFIG.locationConstraints,
replicationEndpoints: [],
};
- const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint,
- replicationConfig);
+ const replicationInfo = _getObjectReplicationInfo(configWithNoReplicationEndpoint, replicationConfig);
assert.deepStrictEqual(replicationInfo, undefined);
});
});
diff --git a/tests/unit/api/apiUtils/object/bumpMicroVersionId.js b/tests/unit/api/apiUtils/object/bumpMicroVersionId.js
new file mode 100644
index 0000000000..2dc50c0515
--- /dev/null
+++ b/tests/unit/api/apiUtils/object/bumpMicroVersionId.js
@@ -0,0 +1,31 @@
+const assert = require('assert');
+
+const bumpMicroVersionId = require('../../../../../lib/api/apiUtils/object/bumpMicroVersionId');
+
+describe('bumpMicroVersionId', () => {
+ it('should set a fresh microVersionId when replicationInfo is present', () => {
+ const objectMD = { replicationInfo: {} };
+ bumpMicroVersionId(objectMD);
+ assert(objectMD.microVersionId, 'expected microVersionId to be set');
+ });
+
+ it('should produce a different value on each call', () => {
+ const objectMD = { replicationInfo: {} };
+ bumpMicroVersionId(objectMD);
+ const first = objectMD.microVersionId;
+ bumpMicroVersionId(objectMD);
+ assert.notStrictEqual(objectMD.microVersionId, first);
+ });
+
+ it('should do nothing when replicationInfo is absent', () => {
+ const objectMD = {};
+ bumpMicroVersionId(objectMD);
+ assert.strictEqual(objectMD.microVersionId, undefined);
+ });
+
+ it('should bump unconditionally when force is true', () => {
+ const objectMD = {};
+ bumpMicroVersionId(objectMD, true);
+ assert(objectMD.microVersionId, 'expected microVersionId to be set when force=true');
+ });
+});
diff --git a/tests/unit/api/apiUtils/versioning.js b/tests/unit/api/apiUtils/versioning.js
index 1edea6d0b0..1bd1e876f0 100644
--- a/tests/unit/api/apiUtils/versioning.js
+++ b/tests/unit/api/apiUtils/versioning.js
@@ -6,11 +6,13 @@ const INF_VID = versioning.VersionID.getInfVid(config.replicationGroupId);
const { scaledMsPerDay } = config.getTimeOptions();
const sinon = require('sinon');
-const { processVersioningState, getMasterState,
- getVersionSpecificMetadataOptions,
- preprocessingVersioningDelete,
- overwritingVersioning } =
- require('../../../../lib/api/apiUtils/object/versioning');
+const {
+ processVersioningState,
+ getMasterState,
+ getVersionSpecificMetadataOptions,
+ preprocessingVersioningDelete,
+ overwritingVersioning,
+} = require('../../../../lib/api/apiUtils/object/versioning');
describe('versioning helpers', () => {
describe('getMasterState+processVersioningState', () => {
@@ -518,17 +520,22 @@ describe('versioning helpers', () => {
},
].forEach(testCase =>
[false, true].forEach(nullVersionCompatMode =>
- ['Enabled', 'Suspended'].forEach(versioningStatus => it(
- `${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}` +
- `, versioning Status=${versioningStatus}`,
- () => {
- const mst = getMasterState(testCase.objMD);
- const res = processVersioningState(mst, versioningStatus, nullVersionCompatMode);
- const resultName = `versioning${versioningStatus}` +
- `${nullVersionCompatMode ? 'Compat' : ''}ExpectedRes`;
- const expectedRes = testCase[resultName];
- assert.deepStrictEqual(res, expectedRes);
- }))));
+ ['Enabled', 'Suspended'].forEach(versioningStatus =>
+ it(
+ `${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}` +
+ `, versioning Status=${versioningStatus}`,
+ () => {
+ const mst = getMasterState(testCase.objMD);
+ const res = processVersioningState(mst, versioningStatus, nullVersionCompatMode);
+ const resultName =
+ `versioning${versioningStatus}` + `${nullVersionCompatMode ? 'Compat' : ''}ExpectedRes`;
+ const expectedRes = testCase[resultName];
+ assert.deepStrictEqual(res, expectedRes);
+ },
+ ),
+ ),
+ ),
+ );
});
describe('getVersionSpecificMetadataOptions', () => {
@@ -583,14 +590,13 @@ describe('versioning helpers', () => {
},
].forEach(testCase =>
[false, true].forEach(nullVersionCompatMode =>
- it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`,
- () => {
- const options = getVersionSpecificMetadataOptions(
- testCase.objMD, nullVersionCompatMode);
- const expectedResAttr = nullVersionCompatMode ?
- 'expectedResCompat' : 'expectedRes';
+ it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, () => {
+ const options = getVersionSpecificMetadataOptions(testCase.objMD, nullVersionCompatMode);
+ const expectedResAttr = nullVersionCompatMode ? 'expectedResCompat' : 'expectedRes';
assert.deepStrictEqual(options, testCase[expectedResAttr]);
- })));
+ }),
+ ),
+ );
});
describe('preprocessingVersioningDelete', () => {
@@ -669,24 +675,28 @@ describe('versioning helpers', () => {
},
].forEach(testCase =>
[false, true].forEach(nullVersionCompatMode =>
- it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`,
- () => {
+ it(`${testCase.description}${nullVersionCompatMode ? ' (null compat)' : ''}`, () => {
const mockBucketMD = {
getVersioningConfiguration: () => ({ Status: 'Enabled' }),
};
const options = preprocessingVersioningDelete(
- 'foobucket', mockBucketMD, testCase.objMD, testCase.reqVersionId,
- nullVersionCompatMode);
- const expectedResAttr = nullVersionCompatMode ?
- 'expectedResCompat' : 'expectedRes';
+ 'foobucket',
+ mockBucketMD,
+ testCase.objMD,
+ testCase.reqVersionId,
+ nullVersionCompatMode,
+ );
+ const expectedResAttr = nullVersionCompatMode ? 'expectedResCompat' : 'expectedRes';
assert.deepStrictEqual(options, testCase[expectedResAttr]);
- })));
+ }),
+ ),
+ );
});
describe('overwritingVersioning', () => {
const days = 3;
const archiveInfo = {
- 'archiveID': '126783123678',
+ archiveID: '126783123678',
};
const now = Date.now();
let clock;
@@ -702,70 +712,70 @@ describe('versioning helpers', () => {
[
{
description: 'Should update archive with restore infos',
- objMD: {
- 'versionId': '2345678',
+ objMD: {
+ versionId: '2345678',
'creation-time': now,
'last-modified': now,
- 'originOp': 's3:PutObject',
+ originOp: 's3:PutObject',
'x-amz-storage-class': 'cold-location',
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'taggingCopy': undefined,
- 'amzStorageClass': 'cold-location',
- 'archive': {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ taggingCopy: undefined,
+ amzStorageClass: 'cold-location',
+ archive: {
archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
- }
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
+ },
},
{
description: 'Should keep user mds and tags',
hasUserMD: true,
objMD: {
- 'versionId': '2345678',
+ versionId: '2345678',
'creation-time': now,
'last-modified': now,
- 'originOp': 's3:PutObject',
+ originOp: 's3:PutObject',
'x-amz-meta-test': 'test',
'x-amz-meta-test2': 'test2',
- 'tags': { 'testtag': 'testtag', 'testtag2': 'testtag2' },
+ tags: { testtag: 'testtag', testtag2: 'testtag2' },
'x-amz-storage-class': 'cold-location',
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'metaHeaders': {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ metaHeaders: {
'x-amz-meta-test': 'test',
'x-amz-meta-test2': 'test2',
},
- 'taggingCopy': { 'testtag': 'testtag', 'testtag2': 'testtag2' },
- 'amzStorageClass': 'cold-location',
- 'archive': {
+ taggingCopy: { testtag: 'testtag', testtag2: 'testtag2' },
+ amzStorageClass: 'cold-location',
+ archive: {
archiveInfo,
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
},
},
{
@@ -773,257 +783,243 @@ describe('versioning helpers', () => {
objMD: {
'creation-time': now,
'last-modified': now,
- 'originOp': 's3:PutObject',
- 'nullVersionId': 'vnull',
- 'isNull': true,
+ originOp: 's3:PutObject',
+ nullVersionId: 'vnull',
+ isNull: true,
'x-amz-storage-class': 'cold-location',
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'amzStorageClass': 'cold-location',
- 'taggingCopy': undefined,
- 'archive': {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ amzStorageClass: 'cold-location',
+ taggingCopy: undefined,
+ archive: {
archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
- }
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
+ },
},
{
description: 'Should not keep x-amz-meta-scal-s3-restore-attempt user MD',
hasUserMD: true,
objMD: {
- 'versionId': '2345678',
+ versionId: '2345678',
'creation-time': now,
'last-modified': now,
- 'originOp': 's3:PutObject',
+ originOp: 's3:PutObject',
'x-amz-meta-test': 'test',
'x-amz-meta-scal-s3-restore-attempt': 14,
'x-amz-storage-class': 'cold-location',
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'metaHeaders': {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ metaHeaders: {
'x-amz-meta-test': 'test',
},
- 'taggingCopy': undefined,
- 'amzStorageClass': 'cold-location',
- 'archive': {
+ taggingCopy: undefined,
+ amzStorageClass: 'cold-location',
+ archive: {
archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
- }
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
+ },
},
{
description: 'Should keep replication infos',
objMD: {
- 'versionId': '2345678',
- 'creation-time': now,
- 'last-modified': now,
- 'originOp': 's3:PutObject',
- 'x-amz-storage-class': 'cold-location',
- 'replicationInfo': {
- 'status': 'COMPLETED',
- 'backends': [
- {
- 'site': 'azure-blob',
- 'status': 'COMPLETED',
- 'dataStoreVersionId': ''
- }
- ],
- 'content': [
- 'DATA',
- 'METADATA'
- ],
- 'destination': 'arn:aws:s3:::replicate-cold',
- 'storageClass': 'azure-blob',
- 'role': 'arn:aws:iam::root:role/s3-replication-role',
- 'storageType': 'azure',
- 'dataStoreVersionId': '',
- },
- archive: {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ versionId: '2345678',
+ 'creation-time': now,
+ 'last-modified': now,
+ originOp: 's3:PutObject',
+ 'x-amz-storage-class': 'cold-location',
+ replicationInfo: {
+ status: 'COMPLETED',
+ backends: [
+ {
+ site: 'azure-blob',
+ status: 'COMPLETED',
+ dataStoreVersionId: '',
+ },
+ ],
+ content: ['DATA', 'METADATA'],
+ destination: 'arn:aws:s3:::replicate-cold',
+ storageClass: 'azure-blob',
+ role: 'arn:aws:iam::root:role/s3-replication-role',
+ storageType: 'azure',
+ dataStoreVersionId: '',
+ },
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'amzStorageClass': 'cold-location',
- 'replicationInfo': {
- 'status': 'COMPLETED',
- 'backends': [
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ amzStorageClass: 'cold-location',
+ replicationInfo: {
+ status: 'COMPLETED',
+ backends: [
{
- 'site': 'azure-blob',
- 'status': 'COMPLETED',
- 'dataStoreVersionId': ''
- }
+ site: 'azure-blob',
+ status: 'COMPLETED',
+ dataStoreVersionId: '',
+ },
],
- 'content': [
- 'DATA',
- 'METADATA'
- ],
- 'destination': 'arn:aws:s3:::replicate-cold',
- 'storageClass': 'azure-blob',
- 'role': 'arn:aws:iam::root:role/s3-replication-role',
- 'storageType': 'azure',
- 'dataStoreVersionId': '',
- },
- 'taggingCopy': undefined,
+ content: ['DATA', 'METADATA'],
+ destination: 'arn:aws:s3:::replicate-cold',
+ storageClass: 'azure-blob',
+ role: 'arn:aws:iam::root:role/s3-replication-role',
+ storageType: 'azure',
+ dataStoreVersionId: '',
+ },
+ taggingCopy: undefined,
archive: {
archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
- }
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
+ },
},
{
description: 'Should keep legalHold',
objMD: {
- 'versionId': '2345678',
- 'creation-time': now,
- 'last-modified': now,
- 'originOp': 's3:PutObject',
- 'legalHold': true,
- 'x-amz-storage-class': 'cold-location',
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ versionId: '2345678',
+ 'creation-time': now,
+ 'last-modified': now,
+ originOp: 's3:PutObject',
+ legalHold: true,
+ 'x-amz-storage-class': 'cold-location',
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'legalHold': true,
- 'amzStorageClass': 'cold-location',
- 'taggingCopy': undefined,
- 'archive': {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ legalHold: true,
+ amzStorageClass: 'cold-location',
+ taggingCopy: undefined,
+ archive: {
archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
- }
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
+ },
},
{
description: 'Should keep ACLs',
objMD: {
- 'versionId': '2345678',
- 'creation-time': now,
- 'last-modified': now,
- 'originOp': 's3:PutObject',
- 'x-amz-storage-class': 'cold-location',
- 'acl': {
- 'Canned': '',
- 'FULL_CONTROL': [
- '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'
- ],
- 'WRITE_ACP': [],
- 'READ': [
- 'http://acs.amazonaws.com/groups/global/AllUsers'
- ],
- 'READ_ACP': []
- },
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
+ versionId: '2345678',
+ 'creation-time': now,
+ 'last-modified': now,
+ originOp: 's3:PutObject',
+ 'x-amz-storage-class': 'cold-location',
+ acl: {
+ Canned: '',
+ FULL_CONTROL: ['872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'],
+ WRITE_ACP: [],
+ READ: ['http://acs.amazonaws.com/groups/global/AllUsers'],
+ READ_ACP: [],
+ },
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
},
expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'acl': {
- 'Canned': '',
- 'FULL_CONTROL': [
- '872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'
- ],
- 'WRITE_ACP': [],
- 'READ': [
- 'http://acs.amazonaws.com/groups/global/AllUsers'
- ],
- 'READ_ACP': []
- },
- 'taggingCopy': undefined,
- 'amzStorageClass': 'cold-location',
- 'archive': {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ acl: {
+ Canned: '',
+ FULL_CONTROL: ['872c04772893deae2b48365752362cd92672eb80eb3deea50d89e834a10ce185'],
+ WRITE_ACP: [],
+ READ: ['http://acs.amazonaws.com/groups/global/AllUsers'],
+ READ_ACP: [],
+ },
+ taggingCopy: undefined,
+ amzStorageClass: 'cold-location',
+ archive: {
archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
},
},
- {
- description: 'Should keep contentMD5 of the original object',
- objMD: {
- 'versionId': '2345678',
+ {
+ description: 'Should keep contentMD5 of the original object',
+ objMD: {
+ versionId: '2345678',
'creation-time': now,
'last-modified': now,
- 'originOp': 's3:PutObject',
+ originOp: 's3:PutObject',
'x-amz-storage-class': 'cold-location',
'content-md5': '123456789-5',
- 'acl': {},
- 'archive': {
- 'restoreRequestedDays': days,
- 'restoreRequestedAt': now,
- archiveInfo
- }
- },
- metadataStoreParams: {
- 'contentMD5': '987654321-3',
- },
- expectedRes: {
- 'creationTime': now,
- 'lastModifiedDate': now,
- 'updateMicroVersionId': true,
- 'originOp': 's3:ObjectRestore:Completed',
- 'contentMD5': '123456789-5',
- 'restoredEtag': '987654321-3',
- 'acl': {},
- 'taggingCopy': undefined,
- 'amzStorageClass': 'cold-location',
- 'archive': {
- archiveInfo,
- 'restoreRequestedDays': 3,
- 'restoreRequestedAt': now,
- 'restoreCompletedAt': new Date(now),
- 'restoreWillExpireAt': new Date(now + (days * scaledMsPerDay)),
- }
- }
+ acl: {},
+ archive: {
+ restoreRequestedDays: days,
+ restoreRequestedAt: now,
+ archiveInfo,
+ },
+ },
+ metadataStoreParams: {
+ contentMD5: '987654321-3',
+ },
+ expectedRes: {
+ creationTime: now,
+ lastModifiedDate: now,
+ updateMicroVersionId: true,
+ originOp: 's3:ObjectRestore:Completed',
+ contentMD5: '123456789-5',
+ restoredEtag: '987654321-3',
+ acl: {},
+ taggingCopy: undefined,
+ amzStorageClass: 'cold-location',
+ archive: {
+ archiveInfo,
+ restoreRequestedDays: 3,
+ restoreRequestedAt: now,
+ restoreCompletedAt: new Date(now),
+ restoreWillExpireAt: new Date(now + days * scaledMsPerDay),
+ },
+ },
},
].forEach(testCase => {
it(testCase.description, () => {
diff --git a/tests/unit/api/objectReplicationMD.js b/tests/unit/api/objectReplicationMD.js
index 48451b43ce..40a02c8c05 100644
--- a/tests/unit/api/objectReplicationMD.js
+++ b/tests/unit/api/objectReplicationMD.js
@@ -1,22 +1,23 @@
const assert = require('assert');
const async = require('async');
const crypto = require('crypto');
+const { promisify } = require('util');
const BucketInfo = require('arsenal').models.BucketInfo;
-const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } =
- require('../helpers');
+const { cleanup, DummyRequestLogger, makeAuthInfo, TaggingConfigTester } = require('../helpers');
const constants = require('../../../constants');
const { metadata } = require('arsenal').storage.metadata.inMemory.metadata;
const DummyRequest = require('../DummyRequest');
const { objectDelete } = require('../../../lib/api/objectDelete');
const objectPut = require('../../../lib/api/objectPut');
const objectCopy = require('../../../lib/api/objectCopy');
-const completeMultipartUpload =
- require('../../../lib/api/completeMultipartUpload');
+const completeMultipartUpload = require('../../../lib/api/completeMultipartUpload');
const objectPutACL = require('../../../lib/api/objectPutACL');
const objectPutTagging = require('../../../lib/api/objectPutTagging');
const objectDeleteTagging = require('../../../lib/api/objectDeleteTagging');
+const objectPutLegalHold = require('../../../lib/api/objectPutLegalHold');
+const objectPutRetention = require('../../../lib/api/objectPutRetention');
const { config } = require('../../../lib/Config');
const log = new DummyRequestLogger();
@@ -55,19 +56,41 @@ const objectACLReq = {
// Get an object request with the given key.
function getObjectPutReq(key, hasContent) {
const bodyContent = hasContent ? 'body content' : '';
- return new DummyRequest({
- bucketName,
- namespace,
- objectKey: key,
- headers: {},
- url: `/${bucketName}/${key}`,
- }, Buffer.from(bodyContent, 'utf8'));
+ return new DummyRequest(
+ {
+ bucketName,
+ namespace,
+ objectKey: key,
+ headers: {},
+ url: `/${bucketName}/${key}`,
+ },
+ Buffer.from(bodyContent, 'utf8'),
+ );
}
-const taggingPutReq = new TaggingConfigTester()
- .createObjectTaggingRequest('PUT', bucketName, keyA);
-const taggingDeleteReq = new TaggingConfigTester()
- .createObjectTaggingRequest('DELETE', bucketName, keyA);
+const taggingPutReq = new TaggingConfigTester().createObjectTaggingRequest('PUT', bucketName, keyA);
+const taggingDeleteReq = new TaggingConfigTester().createObjectTaggingRequest('DELETE', bucketName, keyA);
+
+const legalHoldReq = {
+ bucketName,
+ objectKey: keyA,
+ headers: { host: `${bucketName}.s3.amazonaws.com` },
+ post: '' + 'ON',
+ actionImplicitDenies: false,
+};
+
+const retentionFutureDate = new Date(Date.now() + 7 * 24 * 60 * 60 * 1000).toISOString();
+const retentionReq = {
+ bucketName,
+ objectKey: keyA,
+ headers: { host: `${bucketName}.s3.amazonaws.com` },
+ post:
+ '' +
+ 'GOVERNANCE' +
+ `${retentionFutureDate}` +
+ '',
+ actionImplicitDenies: false,
+};
const emptyReplicationMD = {
status: '',
@@ -94,40 +117,43 @@ const expectedEmptyReplicationMD = {
// Check that the object key has the expected replication information.
function checkObjectReplicationInfo(key, expected) {
const objectMD = metadata.keyMaps.get(bucketName).get(key);
- assert.deepStrictEqual(objectMD.replicationInfo, expected);
+ const actual = { ...objectMD.replicationInfo };
+ const expectedCopy = { ...expected };
+ delete actual.isReplica;
+ delete expectedCopy.isReplica;
+ assert.deepStrictEqual(actual, expectedCopy);
}
// Put the object key and check the replication information.
function putObjectAndCheckMD(key, expected, cb) {
- return objectPut(authInfo, getObjectPutReq(key, true), undefined, log,
- err => {
- if (err) {
- return cb(err);
- }
- checkObjectReplicationInfo(key, expected);
- return cb();
- });
+ return objectPut(authInfo, getObjectPutReq(key, true), undefined, log, err => {
+ if (err) {
+ return cb(err);
+ }
+ checkObjectReplicationInfo(key, expected);
+ return cb();
+ });
}
// Create the bucket in metadata.
function createBucket() {
- metadata
- .buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', ''));
- metadata.keyMaps.set(bucketName, new Map);
+ metadata.buckets.set(bucketName, new BucketInfo(bucketName, ownerID, '', ''));
+ metadata.keyMaps.set(bucketName, new Map());
}
// Create the bucket in metadata with versioning and a replication config.
function createBucketWithReplication(hasStorageClass) {
createBucket();
const config = {
- role: 'arn:aws:iam::account-id:role/src-resource,' +
- 'arn:aws:iam::account-id:role/dest-resource',
+ role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource',
destination: 'arn:aws:s3:::source-bucket',
- rules: [{
- prefix: keyA,
- enabled: true,
- id: 'test-id',
- }],
+ rules: [
+ {
+ prefix: keyA,
+ enabled: true,
+ id: 'test-id',
+ },
+ ],
};
if (hasStorageClass) {
config.rules[0].storageClass = storageClassType;
@@ -140,22 +166,21 @@ function createBucketWithReplication(hasStorageClass) {
// Create the shadow bucket in metadata for MPUs with a recent model number.
function createShadowBucket(key, uploadId) {
- const overviewKey = `overview${constants.splitter}` +
- `${key}${constants.splitter}${uploadId}`;
- metadata.buckets
- .set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', ''));
- // Set modelVersion to use the most recent splitter.
+ const overviewKey = `overview${constants.splitter}` + `${key}${constants.splitter}${uploadId}`;
+ metadata.buckets.set(mpuShadowBucket, new BucketInfo(mpuShadowBucket, ownerID, '', ''));
+ // Set modelVersion to use the most recent splitter.
Object.assign(metadata.buckets.get(mpuShadowBucket), {
_mdBucketModelVersion: 5,
});
- metadata.keyMaps.set(mpuShadowBucket, new Map);
- metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map);
+ metadata.keyMaps.set(mpuShadowBucket, new Map());
+ metadata.keyMaps.get(mpuShadowBucket).set(overviewKey, new Map());
Object.assign(metadata.keyMaps.get(mpuShadowBucket).get(overviewKey), {
id: uploadId,
eventualStorageBucket: bucketName,
initiator: {
DisplayName: 'accessKey1displayName',
- ID: ownerID },
+ ID: ownerID,
+ },
key,
uploadId,
});
@@ -170,24 +195,26 @@ function putMPU(key, body, cb) {
const calculatedHash = md5Hash.digest('hex');
const partKey = `${uploadId}${constants.splitter}00001`;
const obj = {
- partLocations: [{
- key: 1,
- dataStoreName: 'scality-internal-mem',
- dataStoreETag: `1:${calculatedHash}`,
- }],
+ partLocations: [
+ {
+ key: 1,
+ dataStoreName: 'scality-internal-mem',
+ dataStoreETag: `1:${calculatedHash}`,
+ },
+ ],
key: partKey,
};
obj['content-md5'] = calculatedHash;
obj['content-length'] = body.length;
- metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map);
+ metadata.keyMaps.get(mpuShadowBucket).set(partKey, new Map());
const partMap = metadata.keyMaps.get(mpuShadowBucket).get(partKey);
Object.assign(partMap, obj);
const postBody =
'' +
- '' +
- '1' +
- `"${calculatedHash}"` +
- '' +
+ '' +
+ '1' +
+ `"${calculatedHash}"` +
+ '' +
'';
const req = {
bucketName,
@@ -217,8 +244,7 @@ function copyObject(sourceObjectKey, copyObjectKey, hasContent, cb) {
headers: {},
url: `/${bucketName}/${sourceObjectKey}`,
});
- return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined,
- log, cb);
+ return objectCopy(authInfo, req, bucketName, sourceObjectKey, undefined, log, cb);
});
}
@@ -230,26 +256,33 @@ describe('Replication object MD without bucket replication config', () => {
afterEach(() => cleanup());
- it('should not update object metadata', done =>
- putObjectAndCheckMD(keyA, emptyReplicationMD, done));
+ it('should not update object metadata', done => putObjectAndCheckMD(keyA, emptyReplicationMD, done));
it('should not update object metadata if putting object ACL', done =>
- async.series([
- next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
- next => objectPutACL(authInfo, objectACLReq, log, next),
- ], err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD);
- return done();
- }));
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
+ next => objectPutACL(authInfo, objectACLReq, log, next),
+ ],
+ err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD);
+ return done();
+ },
+ ));
describe('Object tagging', () => {
- beforeEach(done => async.series([
- next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
- next => objectPutTagging(authInfo, taggingPutReq, log, next),
- ], err => done(err)));
+ beforeEach(done =>
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
+ next => objectPutTagging(authInfo, taggingPutReq, log, next),
+ ],
+ err => done(err),
+ ),
+ );
it('should not update object metadata if putting tag', done => {
checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD);
@@ -257,18 +290,20 @@ describe('Replication object MD without bucket replication config', () => {
});
it('should not update object metadata if deleting tag', done =>
- async.series([
- // Put a new version to update replication MD content array.
- next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
- next => objectDeleteTagging(authInfo, taggingDeleteReq, log,
- next),
- ], err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD);
- return done();
- }));
+ async.series(
+ [
+ // Put a new version to update replication MD content array.
+ next => putObjectAndCheckMD(keyA, emptyReplicationMD, next),
+ next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next),
+ ],
+ err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, expectedEmptyReplicationMD);
+ return done();
+ },
+ ));
it('should not update object metadata if completing MPU', done =>
putMPU(keyA, 'content', err => {
@@ -291,430 +326,527 @@ describe('Replication object MD without bucket replication config', () => {
});
[true, false].forEach(hasStorageClass => {
- describe('Replication object MD with bucket replication config ' +
- `${hasStorageClass ? 'with' : 'without'} storage class`, () => {
- const replicationMD = {
- status: 'PENDING',
- backends: [{
- site: 'zenko',
+ describe(
+ 'Replication object MD with bucket replication config ' +
+ `${hasStorageClass ? 'with' : 'without'} storage class`,
+ () => {
+ const replicationMD = {
status: 'PENDING',
+ backends: [
+ {
+ site: 'zenko',
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ },
+ ],
+ content: ['DATA', 'METADATA'],
+ destination: bucketARN,
+ storageClass: 'zenko',
+ role: 'arn:aws:iam::account-id:role/src-resource,' + 'arn:aws:iam::account-id:role/dest-resource',
+ storageType: '',
dataStoreVersionId: '',
- }],
- content: ['DATA', 'METADATA'],
- destination: bucketARN,
- storageClass: 'zenko',
- role: 'arn:aws:iam::account-id:role/src-resource,' +
- 'arn:aws:iam::account-id:role/dest-resource',
- storageType: '',
- dataStoreVersionId: '',
- isNFS: undefined,
- };
- const newReplicationMD = hasStorageClass ? Object.assign(replicationMD,
- { storageClass: storageClassType }) : replicationMD;
- const replicateMetadataOnly = Object.assign({}, newReplicationMD,
- { content: ['METADATA'] });
-
- beforeEach(() => {
- cleanup();
- createBucketWithReplication(hasStorageClass);
- });
-
- afterEach(() => {
- cleanup();
- delete config.locationConstraints['zenko'];
- });
-
- it('should update metadata when replication config prefix matches ' +
- 'an object key', done =>
- putObjectAndCheckMD(keyA, newReplicationMD, done));
-
- it('should update metadata when replication config prefix matches ' +
- 'the start of an object key', done =>
- putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done));
-
- it('should not update metadata when replication config prefix does ' +
- 'not match the start of an object key', done =>
- putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done));
-
- it('should not update metadata when replication config prefix does ' +
- 'not apply', done =>
- putObjectAndCheckMD(keyB, emptyReplicationMD, done));
-
- it("should update status to 'PENDING' if putting a new version", done =>
- putObjectAndCheckMD(keyA, newReplicationMD, err => {
- if (err) {
- return done(err);
- }
- const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
- // Update metadata to a status after replication has occurred.
- objectMD.replicationInfo.status = 'COMPLETED';
- return putObjectAndCheckMD(keyA, newReplicationMD, done);
- }));
-
- it("should update status to 'PENDING' and content to '['METADATA']' " +
- 'if putting 0 byte object', done =>
- objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log,
- err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, replicateMetadataOnly);
- return done();
- }));
-
- it('should update metadata if putting object ACL and CRR replication', done => {
- // Set 'zenko' as a typical CRR location (i.e. no type)
- config.locationConstraints['zenko'] = {
- ...config.locationConstraints['zenko'],
- type: '',
+ isNFS: undefined,
};
-
- async.series([
- next => putObjectAndCheckMD(keyA, newReplicationMD, next),
- next => {
- const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
- // Update metadata to a status after replication has occurred.
- objectMD.replicationInfo.status = 'COMPLETED';
- objectPutACL(authInfo, objectACLReq, log, next);
- },
- ], err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, replicateMetadataOnly);
- return done();
+ const newReplicationMD = hasStorageClass
+ ? Object.assign(replicationMD, { storageClass: storageClassType })
+ : replicationMD;
+ const replicateMetadataOnly = Object.assign({}, newReplicationMD, { content: ['METADATA'] });
+
+ beforeEach(() => {
+ cleanup();
+ createBucketWithReplication(hasStorageClass);
});
- });
- it('should not update metadata if putting object ACL and cloud replication', done => {
- // Set 'zenko' as a typical cloud location (i.e. type)
- config.locationConstraints['zenko'] = {
- ...config.locationConstraints['zenko'],
- type: 'aws_s3',
- };
-
- const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' };
-
- let completedReplicationInfo;
- async.series([
- next => putObjectAndCheckMD(keyA, replicationMD, next),
- next => {
- const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
- // Update metadata to a status after replication has occurred.
- objectMD.replicationInfo.status = 'COMPLETED';
- completedReplicationInfo = JSON.parse(
- JSON.stringify(objectMD.replicationInfo));
- objectPutACL(authInfo, objectACLReq, log, next);
- },
- ], err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, completedReplicationInfo);
- return done();
+ afterEach(() => {
+ cleanup();
+ delete config.locationConstraints['zenko'];
});
- });
- it('should update metadata if putting a delete marker', done =>
- async.series([
- next => putObjectAndCheckMD(keyA, newReplicationMD, err => {
- if (err) {
- return next(err);
- }
- const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
- // Set metadata to a status after replication has occurred.
- objectMD.replicationInfo.status = 'COMPLETED';
- return next();
- }),
- next => objectDelete(authInfo, deleteReq, log, next),
- ], err => {
- if (err) {
- return done(err);
- }
- const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
- assert.strictEqual(objectMD.isDeleteMarker, true);
- checkObjectReplicationInfo(keyA, replicateMetadataOnly);
- return done();
- }));
+ it('should update metadata when replication config prefix matches ' + 'an object key', done =>
+ putObjectAndCheckMD(keyA, newReplicationMD, done),
+ );
- it('should not update metadata if putting a delete marker owned by ' +
- 'Lifecycle service account', done =>
- async.series([
- next => putObjectAndCheckMD(keyA, newReplicationMD, next),
- next => objectDelete(authInfoLifecycleService, deleteReq,
- log, next),
- ], err => {
- if (err) {
- return done(err);
- }
- const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
- assert.strictEqual(objectMD.isDeleteMarker, true);
- checkObjectReplicationInfo(keyA, emptyReplicationMD);
- return done();
- }));
-
- describe('Object tagging', () => {
- beforeEach(done => async.series([
- next => putObjectAndCheckMD(keyA, newReplicationMD, next),
- next => objectPutTagging(authInfo, taggingPutReq, log, next),
- ], err => done(err)));
+ it('should update metadata when replication config prefix matches ' + 'the start of an object key', done =>
+ putObjectAndCheckMD(`${keyA}abc`, newReplicationMD, done),
+ );
- it("should update status to 'PENDING' and content to " +
- "'['METADATA']'if putting tag", done => {
- checkObjectReplicationInfo(keyA, replicateMetadataOnly);
- return done();
- });
+ it(
+ 'should not update metadata when replication config prefix does ' +
+ 'not match the start of an object key',
+ done => putObjectAndCheckMD(`abc${keyA}`, emptyReplicationMD, done),
+ );
- it("should update status to 'PENDING' and content to " +
- "'['METADATA']' if deleting tag", done =>
- async.series([
- // Put a new version to update replication MD content array.
- next => putObjectAndCheckMD(keyA, newReplicationMD, next),
- next => objectDeleteTagging(authInfo, taggingDeleteReq, log,
- next),
- ], err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, replicateMetadataOnly);
- return done();
- }));
- });
+ it('should not update metadata when replication config prefix does ' + 'not apply', done =>
+ putObjectAndCheckMD(keyB, emptyReplicationMD, done),
+ );
- describe('Complete MPU', () => {
- it("should update status to 'PENDING' and content to " +
- "'['DATA, METADATA']' if completing MPU", done =>
- putMPU(keyA, 'content', err => {
+ it("should update status to 'PENDING' if putting a new version", done =>
+ putObjectAndCheckMD(keyA, newReplicationMD, err => {
if (err) {
return done(err);
}
- checkObjectReplicationInfo(keyA, newReplicationMD);
- return done();
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ // Update metadata to a status after replication has occurred.
+ objectMD.replicationInfo.status = 'COMPLETED';
+ return putObjectAndCheckMD(keyA, newReplicationMD, done);
}));
- it("should update status to 'PENDING' and content to " +
- "'['METADATA']' if completing MPU with 0 bytes", done =>
- putMPU(keyA, '', err => {
+ it("should update status to 'PENDING' and content to '['METADATA']' " + 'if putting 0 byte object', done =>
+ objectPut(authInfo, getObjectPutReq(keyA, false), undefined, log, err => {
if (err) {
return done(err);
}
checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
- }));
-
- it('should not update replicationInfo if key does not apply',
- done => putMPU(keyB, 'content', err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyB, emptyReplicationMD);
- return done();
- }));
- });
-
- describe('Object copy', () => {
- it("should update status to 'PENDING' and content to " +
- "'['DATA, METADATA']' if copying object", done =>
- copyObject(keyB, keyA, true, err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, newReplicationMD);
- return done();
- }));
+ }),
+ );
- it("should update status to 'PENDING' and content to " +
- "'['METADATA']' if copying object with 0 bytes", done =>
- copyObject(keyB, keyA, false, err => {
- if (err) {
- return done(err);
- }
- checkObjectReplicationInfo(keyA, replicateMetadataOnly);
- return done();
- }));
+ it('should update metadata if putting object ACL and CRR replication', done => {
+ // Set 'zenko' as a typical CRR location (i.e. no type)
+ config.locationConstraints['zenko'] = {
+ ...config.locationConstraints['zenko'],
+ type: '',
+ };
- it('should not update replicationInfo if key does not apply',
- done => {
- const copyKey = `foo-${keyA}`;
- return copyObject(keyB, copyKey, true, err => {
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, newReplicationMD, next),
+ next => {
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ // Update metadata to a status after replication has occurred.
+ objectMD.replicationInfo.status = 'COMPLETED';
+ objectPutACL(authInfo, objectACLReq, log, next);
+ },
+ ],
+ err => {
if (err) {
return done(err);
}
- checkObjectReplicationInfo(copyKey, emptyReplicationMD);
+ checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
- });
- });
- });
+ },
+ );
+ });
- ['awsbackend',
- 'azurebackend',
- 'gcpbackend',
- 'awsbackend,azurebackend'].forEach(backend => {
- const storageTypeMap = {
- 'awsbackend': 'aws_s3',
- 'azurebackend': 'azure',
- 'gcpbackend': 'gcp',
- 'awsbackend,azurebackend': 'aws_s3,azure',
- };
- const storageType = storageTypeMap[backend];
- const backends = backend.split(',').map(site => ({
- site,
- status: 'PENDING',
- dataStoreVersionId: '',
- }));
- describe('Object metadata replicationInfo storageType value',
- () => {
- const expectedReplicationInfo = {
- status: 'PENDING',
- backends,
- content: ['DATA', 'METADATA'],
- destination: 'arn:aws:s3:::destination-bucket',
- storageClass: backend,
- role: 'arn:aws:iam::account-id:role/resource',
- storageType,
- dataStoreVersionId: '',
- isNFS: undefined,
+ it('should not update metadata if putting object ACL and cloud replication', done => {
+ // Set 'zenko' as a typical cloud location (i.e. type)
+ config.locationConstraints['zenko'] = {
+ ...config.locationConstraints['zenko'],
+ type: 'aws_s3',
};
- // Expected for a metadata-only replication operation (for
- // example, putting object tags).
- const expectedReplicationInfoMD = Object.assign({},
- expectedReplicationInfo, { content: ['METADATA'] });
-
- beforeEach(() =>
- // We have already created the bucket, so update the
- // replication configuration to include a location
- // constraint for the `storageClass`. This results in a
- // `storageType` of 'aws_s3', for example.
- Object.assign(metadata.buckets.get(bucketName), {
- _replicationConfiguration: {
- role: 'arn:aws:iam::account-id:role/resource',
- destination: 'arn:aws:s3:::destination-bucket',
- rules: [{
- prefix: keyA,
- enabled: true,
- id: 'test-id',
- storageClass: backend,
- }],
- },
- }));
-
- it('should update on a put object request', done =>
- putObjectAndCheckMD(keyA, expectedReplicationInfo, done));
+ const replicationMD = { ...newReplicationMD, storageType: 'aws_s3' };
- it('should update on a complete MPU object request', done =>
- putMPU(keyA, 'content', err => {
+ let completedReplicationInfo;
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, replicationMD, next),
+ next => {
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ // Update metadata to a status after replication has occurred.
+ objectMD.replicationInfo.status = 'COMPLETED';
+ completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo));
+ objectPutACL(authInfo, objectACLReq, log, next);
+ },
+ ],
+ err => {
if (err) {
return done(err);
}
- const expected =
- Object.assign({}, expectedReplicationInfo,
- { content: ['DATA', 'METADATA', 'MPU'] });
- checkObjectReplicationInfo(keyA, expected);
+ checkObjectReplicationInfo(keyA, completedReplicationInfo);
return done();
- }));
+ },
+ );
+ });
- it('should update on a copy object request', done =>
- copyObject(keyB, keyA, true, err => {
+ it('should update metadata if putting a delete marker', done =>
+ async.series(
+ [
+ next =>
+ putObjectAndCheckMD(keyA, newReplicationMD, err => {
+ if (err) {
+ return next(err);
+ }
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ // Set metadata to a status after replication has occurred.
+ objectMD.replicationInfo.status = 'COMPLETED';
+ return next();
+ }),
+ next => objectDelete(authInfo, deleteReq, log, next),
+ ],
+ err => {
if (err) {
return done(err);
}
- checkObjectReplicationInfo(keyA,
- expectedReplicationInfo);
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ assert.strictEqual(objectMD.isDeleteMarker, true);
+ checkObjectReplicationInfo(keyA, replicateMetadataOnly);
return done();
- }));
-
- it('should update on a put object ACL request', done => {
- let completedReplicationInfo;
- async.series([
- next => putObjectAndCheckMD(keyA,
- expectedReplicationInfo, next),
- next => {
- const objectMD = metadata.keyMaps
- .get(bucketName).get(keyA);
- // Update metadata to a status after replication
- // has occurred.
- objectMD.replicationInfo.status = 'COMPLETED';
- completedReplicationInfo = JSON.parse(
- JSON.stringify(objectMD.replicationInfo));
- objectPutACL(authInfo, objectACLReq, log, next);
- },
- ], err => {
+ },
+ ));
+
+ it('should not update metadata if putting a delete marker owned by ' + 'Lifecycle service account', done =>
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, newReplicationMD, next),
+ next => objectDelete(authInfoLifecycleService, deleteReq, log, next),
+ ],
+ err => {
if (err) {
return done(err);
}
- checkObjectReplicationInfo(keyA, completedReplicationInfo);
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ assert.strictEqual(objectMD.isDeleteMarker, true);
+ checkObjectReplicationInfo(keyA, emptyReplicationMD);
return done();
- });
+ },
+ ),
+ );
+
+ describe('Object tagging', () => {
+ beforeEach(done =>
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, newReplicationMD, next),
+ next => objectPutTagging(authInfo, taggingPutReq, log, next),
+ ],
+ err => done(err),
+ ),
+ );
+
+ it("should update status to 'PENDING' and content to " + "'['METADATA']'if putting tag", done => {
+ checkObjectReplicationInfo(keyA, replicateMetadataOnly);
+ return done();
});
- it('should update on a put object tagging request', done =>
- async.series([
- next => putObjectAndCheckMD(keyA,
- expectedReplicationInfo, next),
- next => objectPutTagging(authInfo, taggingPutReq, log,
- next),
- ], err => {
+ it("should update status to 'PENDING' and content to " + "'['METADATA']' if deleting tag", done =>
+ async.series(
+ [
+ // Put a new version to update replication MD content array.
+ next => putObjectAndCheckMD(keyA, newReplicationMD, next),
+ next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next),
+ ],
+ err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, replicateMetadataOnly);
+ return done();
+ },
+ ),
+ );
+ });
+
+ describe('Complete MPU', () => {
+ it(
+ "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if completing MPU",
+ done =>
+ putMPU(keyA, 'content', err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, newReplicationMD);
+ return done();
+ }),
+ );
+
+ it(
+ "should update status to 'PENDING' and content to " +
+ "'['METADATA']' if completing MPU with 0 bytes",
+ done =>
+ putMPU(keyA, '', err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, replicateMetadataOnly);
+ return done();
+ }),
+ );
+
+ it('should not update replicationInfo if key does not apply', done =>
+ putMPU(keyB, 'content', err => {
if (err) {
return done(err);
}
- const expected = Object.assign({},
- expectedReplicationInfo,
- { content: ['METADATA', 'PUT_TAGGING'] });
- checkObjectReplicationInfo(keyA, expected);
+ checkObjectReplicationInfo(keyB, emptyReplicationMD);
return done();
}));
+ });
- it('should update on a delete tagging request', done =>
- async.series([
- next => putObjectAndCheckMD(keyA,
- expectedReplicationInfo, next),
- next => objectDeleteTagging(authInfo, taggingDeleteReq,
- log, next),
- ], err => {
+ describe('Object copy', () => {
+ it(
+ "should update status to 'PENDING' and content to " + "'['DATA, METADATA']' if copying object",
+ done =>
+ copyObject(keyB, keyA, true, err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, newReplicationMD);
+ return done();
+ }),
+ );
+
+ it(
+ "should update status to 'PENDING' and content to " +
+ "'['METADATA']' if copying object with 0 bytes",
+ done =>
+ copyObject(keyB, keyA, false, err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, replicateMetadataOnly);
+ return done();
+ }),
+ );
+
+ it('should not update replicationInfo if key does not apply', done => {
+ const copyKey = `foo-${keyA}`;
+ return copyObject(keyB, copyKey, true, err => {
if (err) {
return done(err);
}
- const expected = Object.assign({},
- expectedReplicationInfo,
- { content: ['METADATA', 'DELETE_TAGGING'] });
- checkObjectReplicationInfo(keyA, expected);
+ checkObjectReplicationInfo(copyKey, emptyReplicationMD);
return done();
- }));
+ });
+ });
+ });
+
+ ['awsbackend', 'azurebackend', 'gcpbackend', 'awsbackend,azurebackend'].forEach(backend => {
+ const storageTypeMap = {
+ awsbackend: 'aws_s3',
+ azurebackend: 'azure',
+ gcpbackend: 'gcp',
+ 'awsbackend,azurebackend': 'aws_s3,azure',
+ };
+ const storageType = storageTypeMap[backend];
+ const backends = backend.split(',').map(site => ({
+ site,
+ status: 'PENDING',
+ dataStoreVersionId: '',
+ }));
+ describe('Object metadata replicationInfo storageType value', () => {
+ const expectedReplicationInfo = {
+ status: 'PENDING',
+ backends,
+ content: ['DATA', 'METADATA'],
+ destination: 'arn:aws:s3:::destination-bucket',
+ storageClass: backend,
+ role: 'arn:aws:iam::account-id:role/resource',
+ storageType,
+ dataStoreVersionId: '',
+ isNFS: undefined,
+ };
+
+ // Expected for a metadata-only replication operation (for
+ // example, putting object tags).
+ const expectedReplicationInfoMD = Object.assign({}, expectedReplicationInfo, {
+ content: ['METADATA'],
+ });
- it('should update when putting a delete marker', done =>
- async.series([
- next => putObjectAndCheckMD(keyA,
- expectedReplicationInfo, err => {
+ beforeEach(() =>
+ // We have already created the bucket, so update the
+ // replication configuration to include a location
+ // constraint for the `storageClass`. This results in a
+ // `storageType` of 'aws_s3', for example.
+ Object.assign(metadata.buckets.get(bucketName), {
+ _replicationConfiguration: {
+ role: 'arn:aws:iam::account-id:role/resource',
+ destination: 'arn:aws:s3:::destination-bucket',
+ rules: [
+ {
+ prefix: keyA,
+ enabled: true,
+ id: 'test-id',
+ storageClass: backend,
+ },
+ ],
+ },
+ }),
+ );
+
+ it('should update on a put object request', done =>
+ putObjectAndCheckMD(keyA, expectedReplicationInfo, done));
+
+ it('should update on a complete MPU object request', done =>
+ putMPU(keyA, 'content', err => {
+ if (err) {
+ return done(err);
+ }
+ const expected = Object.assign({}, expectedReplicationInfo, {
+ content: ['DATA', 'METADATA', 'MPU'],
+ });
+ checkObjectReplicationInfo(keyA, expected);
+ return done();
+ }));
+
+ it('should update on a copy object request', done =>
+ copyObject(keyB, keyA, true, err => {
+ if (err) {
+ return done(err);
+ }
+ checkObjectReplicationInfo(keyA, expectedReplicationInfo);
+ return done();
+ }));
+
+ it('should update on a put object ACL request', done => {
+ let completedReplicationInfo;
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next),
+ next => {
+ const objectMD = metadata.keyMaps.get(bucketName).get(keyA);
+ // Update metadata to a status after replication
+ // has occurred.
+ objectMD.replicationInfo.status = 'COMPLETED';
+ completedReplicationInfo = JSON.parse(JSON.stringify(objectMD.replicationInfo));
+ objectPutACL(authInfo, objectACLReq, log, next);
+ },
+ ],
+ err => {
if (err) {
- return next(err);
+ return done(err);
}
- // Update metadata to a status indicating that
- // replication has occurred for the object.
- metadata
- .keyMaps
- .get(bucketName)
- .get(keyA)
- .replicationInfo
- .status = 'COMPLETED';
- return next();
- }),
- next => objectDelete(authInfo, deleteReq, log, next),
- ], err => {
- if (err) {
- return done(err);
- }
- // Is it, in fact, a delete marker?
- assert(metadata
- .keyMaps
- .get(bucketName)
- .get(keyA)
- .isDeleteMarker);
- checkObjectReplicationInfo(keyA,
- expectedReplicationInfoMD);
- return done();
- }));
+ checkObjectReplicationInfo(keyA, completedReplicationInfo);
+ return done();
+ },
+ );
+ });
+
+ it('should update on a put object tagging request', done =>
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next),
+ next => objectPutTagging(authInfo, taggingPutReq, log, next),
+ ],
+ err => {
+ if (err) {
+ return done(err);
+ }
+ const expected = Object.assign({}, expectedReplicationInfo, {
+ content: ['METADATA', 'PUT_TAGGING'],
+ });
+ checkObjectReplicationInfo(keyA, expected);
+ return done();
+ },
+ ));
+
+ it('should update on a delete tagging request', done =>
+ async.series(
+ [
+ next => putObjectAndCheckMD(keyA, expectedReplicationInfo, next),
+ next => objectDeleteTagging(authInfo, taggingDeleteReq, log, next),
+ ],
+ err => {
+ if (err) {
+ return done(err);
+ }
+ const expected = Object.assign({}, expectedReplicationInfo, {
+ content: ['METADATA', 'DELETE_TAGGING'],
+ });
+ checkObjectReplicationInfo(keyA, expected);
+ return done();
+ },
+ ));
+
+ it('should update when putting a delete marker', done =>
+ async.series(
+ [
+ next =>
+ putObjectAndCheckMD(keyA, expectedReplicationInfo, err => {
+ if (err) {
+ return next(err);
+ }
+ // Update metadata to a status indicating that
+ // replication has occurred for the object.
+ metadata.keyMaps.get(bucketName).get(keyA).replicationInfo.status = 'COMPLETED';
+ return next();
+ }),
+ next => objectDelete(authInfo, deleteReq, log, next),
+ ],
+ err => {
+ if (err) {
+ return done(err);
+ }
+ // Is it, in fact, a delete marker?
+ assert(metadata.keyMaps.get(bucketName).get(keyA).isDeleteMarker);
+ checkObjectReplicationInfo(keyA, expectedReplicationInfoMD);
+ return done();
+ },
+ ));
+ });
+ });
+ },
+ );
+});
+
+const metadataOnlyWrites = [
+ { name: 'objectPutTagging', fn: objectPutTagging, req: taggingPutReq },
+ { name: 'objectDeleteTagging', fn: objectDeleteTagging, req: taggingDeleteReq },
+ { name: 'objectPutACL', fn: objectPutACL, req: objectACLReq, skipSubsequentBump: true },
+ { name: 'objectPutLegalHold', fn: objectPutLegalHold, req: legalHoldReq, requiresObjectLock: true },
+ { name: 'objectPutRetention', fn: objectPutRetention, req: retentionReq, requiresObjectLock: true },
+];
+
+describe('microVersionId is bumped on metadata-only writes', () => {
+ const getMD = key => metadata.keyMaps.get(bucketName).get(key);
+ const objectPutAsync = promisify(objectPut);
+
+ beforeEach(() => {
+ cleanup();
+ createBucketWithReplication(true);
+ metadata.buckets.get(bucketName).setObjectLockEnabled(true);
+ });
+
+ afterEach(() => cleanup());
+
+ metadataOnlyWrites.forEach(({ name, fn, req, skipSubsequentBump }) => {
+ const asyncFn = promisify(fn);
+
+ it(`should set microVersionId on ${name}`, async () => {
+ await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log);
+ await asyncFn(authInfo, req, log);
+ assert(getMD(keyA).microVersionId, `expected microVersionId to be set after ${name}`);
+ });
+
+ if (!skipSubsequentBump) {
+ it(`should bump microVersionId on subsequent ${name}`, async () => {
+ await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log);
+ await asyncFn(authInfo, req, log);
+ const before = getMD(keyA).microVersionId;
+ await asyncFn(authInfo, req, log);
+ const after = getMD(keyA).microVersionId;
+ assert(after && after !== before, `expected microVersionId to change after ${name}`);
});
+ }
+ });
+});
+
+describe('isReplica is cleared on direct user writes overwriting a replica', () => {
+ const getMD = key => metadata.keyMaps.get(bucketName).get(key);
+ const objectPutAsync = promisify(objectPut);
+
+ beforeEach(() => {
+ cleanup();
+ createBucketWithReplication(true);
+ metadata.buckets.get(bucketName).setObjectLockEnabled(true);
+ });
+
+ afterEach(() => cleanup());
+
+ metadataOnlyWrites.forEach(({ name, fn, req }) => {
+ const asyncFn = promisify(fn);
+
+ it(`should clear isReplica on ${name} when prior MD has it true`, async () => {
+ await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log);
+ getMD(keyA).replicationInfo.isReplica = true;
+ await asyncFn(authInfo, req, log);
+ assert.strictEqual(getMD(keyA).replicationInfo.isReplica, false);
+ });
+
+ it(`should leave isReplica false on ${name} when prior MD does not have it`, async () => {
+ await objectPutAsync(authInfo, getObjectPutReq(keyA, true), undefined, log);
+ await asyncFn(authInfo, req, log);
+ assert.strictEqual(getMD(keyA).replicationInfo.isReplica, false);
});
});
});
diff --git a/tests/unit/lib/services.spec.js b/tests/unit/lib/services.spec.js
index f76ee39502..ba8b4499b2 100644
--- a/tests/unit/lib/services.spec.js
+++ b/tests/unit/lib/services.spec.js
@@ -94,8 +94,12 @@ describe('services', () => {
IsTruncated: false,
});
- services.findObjectVersionByUploadId(bucketName, objectKey, 'non-existent-upload-id',
- log, (err, foundVersion) => {
+ services.findObjectVersionByUploadId(
+ bucketName,
+ objectKey,
+ 'non-existent-upload-id',
+ log,
+ (err, foundVersion) => {
assert.ifError(err);
sinon.assert.calledTwice(getObjectListingStub);
@@ -104,7 +108,8 @@ describe('services', () => {
assert.strictEqual(secondCallParams.versionIdMarker, 'version-marker');
assert.strictEqual(foundVersion, null);
done();
- });
+ },
+ );
});
it('should find a version on the first page of many and stop listing', done => {
@@ -176,10 +181,10 @@ describe('services', () => {
let putObjectMDStub;
beforeEach(() => {
- putObjectMDStub = sinon.stub(metadata, 'putObjectMD')
+ putObjectMDStub = sinon
+ .stub(metadata, 'putObjectMD')
.callsFake((bucket, key, md, opts, reqLog, cb) => cb(null));
- sinon.stub(acl, 'parseAclFromHeaders')
- .callsFake((params, cb) => cb(null, { Canned: 'private' }));
+ sinon.stub(acl, 'parseAclFromHeaders').callsFake((params, cb) => cb(null, { Canned: 'private' }));
});
it('should store checksumAlgorithm, checksumType and checksumIsDefault when provided', done => {
@@ -241,8 +246,7 @@ describe('services', () => {
const authInfo = makeAuthInfo('accessKey1');
const ownerID = authInfo.getCanonicalID();
const mpuBucketName = `${constants.mpuBucketPrefix}${bucketName}`;
- const mpuOverviewKey = `overview${constants.splitter}${objectKey}` +
- `${constants.splitter}${uploadId}`;
+ const mpuOverviewKey = `overview${constants.splitter}${objectKey}` + `${constants.splitter}${uploadId}`;
const mpuBucket = {
getName: () => mpuBucketName,
getMdBucketModelVersion: () => 2,
@@ -268,46 +272,48 @@ describe('services', () => {
assert.strictEqual(name, mpuBucketName);
done(null, mpuBucket);
});
- sinon.stub(metadata, 'getObjectMD')
- .callsFake((bucket, key, params, reqLog, done) => {
- assert.strictEqual(bucket, mpuBucketName);
- assert.strictEqual(key, mpuOverviewKey);
- done(null, {
- ...storedMetadata,
- ...storedMetadataOverride,
- });
+ sinon.stub(metadata, 'getObjectMD').callsFake((bucket, key, params, reqLog, done) => {
+ assert.strictEqual(bucket, mpuBucketName);
+ assert.strictEqual(key, mpuOverviewKey);
+ done(null, {
+ ...storedMetadata,
+ ...storedMetadataOverride,
});
+ });
- services.metadataValidateMultipart({
- bucketName,
- objectKey,
- uploadId,
- authInfo,
- requestType: 'listParts',
- log,
- }, cb);
+ services.metadataValidateMultipart(
+ {
+ bucketName,
+ objectKey,
+ uploadId,
+ authInfo,
+ requestType: 'listParts',
+ log,
+ },
+ cb,
+ );
}
- it('should expose checksum fields from stored MPU overview metadata',
- done => {
- validateMultipart({
- checksumAlgorithm: 'sha256',
- checksumType: 'COMPOSITE',
- checksumIsDefault: false,
- }, (err, bucket, mpuOverview, returnedStoredMetadata) => {
- assert.ifError(err);
- assert.strictEqual(bucket, mpuBucket);
- assert.strictEqual(returnedStoredMetadata.checksumAlgorithm,
- 'sha256');
- assert.strictEqual(mpuOverview.checksumAlgorithm, 'sha256');
- assert.strictEqual(mpuOverview.checksumType, 'COMPOSITE');
- assert.strictEqual(mpuOverview.checksumIsDefault, false);
- done();
- });
+ it('should expose checksum fields from stored MPU overview metadata', done => {
+ validateMultipart(
+ {
+ checksumAlgorithm: 'sha256',
+ checksumType: 'COMPOSITE',
+ checksumIsDefault: false,
+ },
+ (err, bucket, mpuOverview, returnedStoredMetadata) => {
+ assert.ifError(err);
+ assert.strictEqual(bucket, mpuBucket);
+ assert.strictEqual(returnedStoredMetadata.checksumAlgorithm, 'sha256');
+ assert.strictEqual(mpuOverview.checksumAlgorithm, 'sha256');
+ assert.strictEqual(mpuOverview.checksumType, 'COMPOSITE');
+ assert.strictEqual(mpuOverview.checksumIsDefault, false);
+ done();
+ },
+ );
});
- it('should leave checksum fields undefined for legacy MPU overview metadata',
- done => {
+ it('should leave checksum fields undefined for legacy MPU overview metadata', done => {
validateMultipart({}, (err, bucket, mpuOverview) => {
assert.ifError(err);
assert.strictEqual(bucket, mpuBucket);
@@ -318,4 +324,39 @@ describe('services', () => {
});
});
});
+
+ describe('metadataStoreObject', () => {
+ const authInfo = makeAuthInfo('accessKey1');
+ const params = {
+ objectKey,
+ authInfo,
+ size: 0,
+ contentMD5: 'd41d8cd98f00b204e9800998ecf8427e',
+ metaHeaders: {},
+ headers: {},
+ log,
+ };
+
+ beforeEach(() => {
+ sinon.stub(metadata, 'putObjectMD').yields(null);
+ });
+
+ it('should set microVersionId on the stored MD when updateMicroVersionId is true', done => {
+ services.metadataStoreObject(bucketName, null, null, { ...params, updateMicroVersionId: true }, err => {
+ assert.ifError(err);
+ const storedMD = metadata.putObjectMD.firstCall.args[2];
+ assert(storedMD.getMicroVersionId(), 'expected microVersionId to be set on stored MD');
+ done();
+ });
+ });
+
+ it('should not set microVersionId when updateMicroVersionId is not set', done => {
+ services.metadataStoreObject(bucketName, null, null, params, err => {
+ assert.ifError(err);
+ const storedMD = metadata.putObjectMD.firstCall.args[2];
+ assert.strictEqual(storedMD.getMicroVersionId(), undefined);
+ done();
+ });
+ });
+ });
});
diff --git a/tests/unit/utils/collectResponseHeaders.js b/tests/unit/utils/collectResponseHeaders.js
index 75cd3c134f..de17b64321 100644
--- a/tests/unit/utils/collectResponseHeaders.js
+++ b/tests/unit/utils/collectResponseHeaders.js
@@ -1,6 +1,5 @@
const assert = require('assert');
-const collectResponseHeaders =
- require('../../../lib/utilities/collectResponseHeaders');
+const collectResponseHeaders = require('../../../lib/utilities/collectResponseHeaders');
describe('Middleware: Collect Response Headers', () => {
it('should be able to set replication status when config is set', () => {
@@ -9,6 +8,22 @@ describe('Middleware: Collect Response Headers', () => {
assert.deepStrictEqual(headers['x-amz-replication-status'], 'REPLICA');
});
+ it('should set REPLICA header from isReplica even when status is PENDING', () => {
+ const objectMD = {
+ replicationInfo: { status: 'PENDING', isReplica: true },
+ };
+ const headers = collectResponseHeaders(objectMD);
+ assert.deepStrictEqual(headers['x-amz-replication-status'], 'REPLICA');
+ });
+
+ it('should use replicationInfo.status when isReplica is false', () => {
+ const objectMD = {
+ replicationInfo: { status: 'PENDING', isReplica: false },
+ };
+ const headers = collectResponseHeaders(objectMD);
+ assert.deepStrictEqual(headers['x-amz-replication-status'], 'PENDING');
+ });
+
it('should set the replication status of each site', () => {
const objectMD = {
replicationInfo: {
@@ -21,22 +36,19 @@ describe('Middleware: Collect Response Headers', () => {
};
const headers = collectResponseHeaders(objectMD);
assert.deepStrictEqual(headers['x-amz-replication-status'], 'COMPLETED');
- assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'],
- 'COMPLETED');
+ assert.deepStrictEqual(headers['x-amz-meta-us-east-1-replication-status'], 'COMPLETED');
assert.deepStrictEqual(headers['x-amz-meta-us-east-1-version-id'], '123');
- assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'],
- 'COMPLETED');
+ assert.deepStrictEqual(headers['x-amz-meta-us-west-2-replication-status'], 'COMPLETED');
assert.deepStrictEqual(headers['x-amz-meta-us-west-2-version-id'], undefined);
});
-
+
[
{ md: { replicationInfo: null }, test: 'when config is not set' },
{ md: {}, test: 'for older objects' },
].forEach(item => {
it(`should skip replication header ${item.test}`, () => {
const headers = collectResponseHeaders(item.md);
- assert.deepStrictEqual(headers['x-amz-replication-status'],
- undefined);
+ assert.deepStrictEqual(headers['x-amz-replication-status'], undefined);
});
});
@@ -45,19 +57,16 @@ describe('Middleware: Collect Response Headers', () => {
assert.strictEqual(headers['Accept-Ranges'], 'bytes');
});
- it('should return an undefined value when x-amz-website-redirect-location' +
- ' is empty', () => {
+ it('should return an undefined value when x-amz-website-redirect-location' + ' is empty', () => {
const objectMD = { 'x-amz-website-redirect-location': '' };
const headers = collectResponseHeaders(objectMD);
- assert.strictEqual(headers['x-amz-website-redirect-location'],
- undefined);
+ assert.strictEqual(headers['x-amz-website-redirect-location'], undefined);
});
it('should return the (nonempty) value of WebsiteRedirectLocation', () => {
const obj = { 'x-amz-website-redirect-location': 'google.com' };
const headers = collectResponseHeaders(obj);
- assert.strictEqual(headers['x-amz-website-redirect-location'],
- 'google.com');
+ assert.strictEqual(headers['x-amz-website-redirect-location'], 'google.com');
});
it('should not set flag when transition not in progress', () => {
diff --git a/yarn.lock b/yarn.lock
index 6a6ab403de..73feb78843 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -6065,32 +6065,38 @@ arraybuffer.prototype.slice@^1.0.4:
get-intrinsic "^1.2.6"
is-array-buffer "^3.0.4"
-"arsenal@git+https://github.com/scality/Arsenal#8.2.28":
- version "8.2.28"
- resolved "git+https://github.com/scality/Arsenal#7df5088715bb26a62ff1db2045e611029ff17de1"
+"arsenal@git+https://github.com/scality/Arsenal#2c429ab35a5ac82c3dafa5a0296a49a23a9c8a4a":
+ version "8.4.4"
+ resolved "git+https://github.com/scality/Arsenal#2c429ab35a5ac82c3dafa5a0296a49a23a9c8a4a"
dependencies:
- "@azure/identity" "^4.10.2"
- "@azure/storage-blob" "^12.27.0"
+ "@aws-sdk/client-kms" "^3.975.0"
+ "@aws-sdk/client-s3" "^3.975.0"
+ "@aws-sdk/credential-providers" "^3.975.0"
+ "@aws-sdk/lib-storage" "^3.975.0"
+ "@azure/identity" "^4.13.0"
+ "@azure/storage-blob" "^12.31.0"
"@js-sdsl/ordered-set" "^4.4.2"
- "@scality/hdclient" "^1.3.1"
+ "@opentelemetry/api" "^1.9.0"
+ "@scality/hdclient" "^1.3.2"
+ "@smithy/node-http-handler" "^4.3.0"
+ "@smithy/protocol-http" "^5.3.5"
JSONStream "^1.3.5"
agentkeepalive "^4.6.0"
ajv "6.12.3"
async "~2.6.4"
- aws-sdk "^2.1691.0"
backo "^1.1.0"
base-x "3.0.8"
base62 "^2.0.2"
- debug "^4.4.1"
+ debug "^4.4.3"
fcntl "github:scality/node-fcntl#0.3.0"
httpagent scality/httpagent#1.1.0
https-proxy-agent "^7.0.6"
- ioredis "^5.6.1"
+ ioredis "^5.8.1"
ipaddr.js "^2.2.0"
- joi "^17.13.3"
+ joi "^18.0.1"
level "~5.0.1"
level-sublevel "~6.6.5"
- mongodb "^6.17.0"
+ mongodb "^6.20.0"
node-forge "^1.3.1"
prom-client "^15.1.3"
simple-glob "^0.2.0"
@@ -6104,37 +6110,32 @@ arraybuffer.prototype.slice@^1.0.4:
optionalDependencies:
ioctl "^2.0.2"
-"arsenal@git+https://github.com/scality/Arsenal#8.2.4":
- version "8.2.4"
- resolved "git+https://github.com/scality/Arsenal#96ef6a3e26d7528f877300606586759f1da6d0cd"
+"arsenal@git+https://github.com/scality/Arsenal#8.2.28":
+ version "8.2.28"
+ resolved "git+https://github.com/scality/Arsenal#7df5088715bb26a62ff1db2045e611029ff17de1"
dependencies:
- "@azure/identity" "^4.5.0"
- "@azure/storage-blob" "^12.25.0"
- "@eslint/plugin-kit" "^0.2.3"
+ "@azure/identity" "^4.10.2"
+ "@azure/storage-blob" "^12.27.0"
"@js-sdsl/ordered-set" "^4.4.2"
"@scality/hdclient" "^1.3.1"
- "@types/async" "^3.2.24"
- "@types/utf8" "^3.0.3"
JSONStream "^1.3.5"
- agentkeepalive "^4.5.0"
+ agentkeepalive "^4.6.0"
ajv "6.12.3"
async "~2.6.4"
aws-sdk "^2.1691.0"
backo "^1.1.0"
base-x "3.0.8"
base62 "^2.0.2"
- bson "^6.8.0"
- debug "^4.3.7"
- diskusage "^1.2.0"
+ debug "^4.4.1"
fcntl "github:scality/node-fcntl#0.3.0"
httpagent scality/httpagent#1.1.0
- https-proxy-agent "^7.0.5"
- ioredis "^5.4.1"
+ https-proxy-agent "^7.0.6"
+ ioredis "^5.6.1"
ipaddr.js "^2.2.0"
joi "^17.13.3"
level "~5.0.1"
level-sublevel "~6.6.5"
- mongodb "^6.11.0"
+ mongodb "^6.17.0"
node-forge "^1.3.1"
prom-client "^15.1.3"
simple-glob "^0.2.0"
@@ -6148,38 +6149,37 @@ arraybuffer.prototype.slice@^1.0.4:
optionalDependencies:
ioctl "^2.0.2"
-"arsenal@git+https://github.com/scality/Arsenal#8.4.4":
- version "8.4.4"
- resolved "git+https://github.com/scality/Arsenal#a1fa5b412c0f634fa87fc7ccd7b086973d2f8d87"
+"arsenal@git+https://github.com/scality/Arsenal#8.2.4":
+ version "8.2.4"
+ resolved "git+https://github.com/scality/Arsenal#96ef6a3e26d7528f877300606586759f1da6d0cd"
dependencies:
- "@aws-sdk/client-kms" "^3.975.0"
- "@aws-sdk/client-s3" "^3.975.0"
- "@aws-sdk/credential-providers" "^3.975.0"
- "@aws-sdk/lib-storage" "^3.975.0"
- "@azure/identity" "^4.13.0"
- "@azure/storage-blob" "^12.31.0"
+ "@azure/identity" "^4.5.0"
+ "@azure/storage-blob" "^12.25.0"
+ "@eslint/plugin-kit" "^0.2.3"
"@js-sdsl/ordered-set" "^4.4.2"
- "@opentelemetry/api" "^1.9.0"
- "@scality/hdclient" "^1.3.2"
- "@smithy/node-http-handler" "^4.3.0"
- "@smithy/protocol-http" "^5.3.5"
+ "@scality/hdclient" "^1.3.1"
+ "@types/async" "^3.2.24"
+ "@types/utf8" "^3.0.3"
JSONStream "^1.3.5"
- agentkeepalive "^4.6.0"
+ agentkeepalive "^4.5.0"
ajv "6.12.3"
async "~2.6.4"
+ aws-sdk "^2.1691.0"
backo "^1.1.0"
base-x "3.0.8"
base62 "^2.0.2"
- debug "^4.4.3"
+ bson "^6.8.0"
+ debug "^4.3.7"
+ diskusage "^1.2.0"
fcntl "github:scality/node-fcntl#0.3.0"
httpagent scality/httpagent#1.1.0
- https-proxy-agent "^7.0.6"
- ioredis "^5.8.1"
+ https-proxy-agent "^7.0.5"
+ ioredis "^5.4.1"
ipaddr.js "^2.2.0"
- joi "^18.0.1"
+ joi "^17.13.3"
level "~5.0.1"
level-sublevel "~6.6.5"
- mongodb "^6.20.0"
+ mongodb "^6.11.0"
node-forge "^1.3.1"
prom-client "^15.1.3"
simple-glob "^0.2.0"