S3UTILS-217: improve listObjectsByReplicationStatus.js

listObjectsByReplicationStatus.js

@@ -37,6 +37,7 @@ function _getKeys(list) {
     return list.map(v => ({
         Key: v.Key,
         VersionId: v.VersionId,
+        IsLatest: v.IsLatest,
     }));
 }
 
@@ -65,7 +66,7 @@ function _listBucket(s3, log, replicationStatusToProcess, bucket, cb) {
                 }
                 const keys = _getKeys(data.Versions || []);
                 return async.mapLimit(keys, 10, (k, next) => {
-                    const { Key, VersionId } = k;
+                    const { Key, VersionId, IsLatest } = k;
                     s3.send(new HeadObjectCommand({
                         Bucket: bucketName,
                         Key,
@@ -74,14 +75,30 @@ function _listBucket(s3, log, replicationStatusToProcess, bucket, cb) {
                         if (replicationStatusToProcess.includes(res.ReplicationStatus)) {
                             log.info('object with matching replication status found', {
                                 Key,
+                                VersionId,
+                                IsLatest,
                                 ReplicationStatus: res.ReplicationStatus,
-                                ...res
+                                ...res,
+                                bucketName
                             });
                         }
                         return next();
-                    }).catch(next);
+                    }).catch(err => {
+                        log.error('error getting object metadata', {
+                            bucketName,
+                            Key,
+                            VersionId,
+                            IsLatest,
+                            error: err
+                        });
+                        return next();
+                    });
                 }, err => {
                     if (err) {
+                        log.error('error processing batch of objects', {
+                            error: err,
+                            bucketName
+                        });
                         return done(err);
                     }
                     VersionIdMarker = data.NextVersionIdMarker;

tests/functional/listObjectsByReplicationStatus.js

@@ -203,9 +203,6 @@ describe('listObjectsByReplicationStatus', () => {
             adminAccessKeyId,
             adminSecretAccessKey
         );
-    });
-
-    beforeEach(async () => {
         log.info('Setting up test accounts and buckets');
         accountSource = await createTestAccount(vaultClient);
         accountDest = await createTestAccount(vaultClient);
@@ -214,7 +211,7 @@ describe('listObjectsByReplicationStatus', () => {
         await configureCrr(accountSource, accountDest);
     });
 
-    afterEach(async () => {
+    afterAll(async () => {
         log.info('Cleaning up test accounts');
         await removeCrrConfiguration(accountSource);
         await removeCrrConfiguration(accountDest);
@@ -223,6 +220,46 @@ describe('listObjectsByReplicationStatus', () => {
         log.info('Test accounts deleted');
     });
 
+    it('should log an error when using credentials without access to the bucket', async () => {
+        // Create a test user with no bucket permissions
+        const noPermAccount = await createTestAccount(vaultClient);
+
+        log.info('Testing listObjectsByReplicationStatus with unauthorized key pair');
+
+        // Create a custom logger to capture logs
+
+        const capturedLogs = [];
+        const captureLogger = new Logger('s3utils:listObjectsByReplicationStatus:test');
+        const originalInfo = captureLogger.info.bind(captureLogger);
+        captureLogger.info = function (message, data) {
+            capturedLogs.push({ message, data });
+            return originalInfo(message, data);
+        };
+
+        const endpoint = `http://${s3Host}:${s3Port}`;
+        try {
+            // Attempt to list objects by replication status with no permissions
+            await listObjectsByReplicationStatus({
+                buckets: accountSource.bucketName,
+                accessKey: noPermAccount.accountAccessKey,
+                secretKey: noPermAccount.accountSecretKey,
+                endpoint,
+                replicationStatus: 'PENDING', // Choose a valid status
+                log: captureLogger,
+            });
+        } catch (err) {
+            // The function may throw, but regardless we want to inspect the logs
+        } finally {
+            // Clean up the account
+            await deleteTestAccount(vaultClient, noPermAccount);
+        }
+
+        // The logs should contain at least one authorization error
+        const authErrorLog = capturedLogs.filter(entry => entry.name === 'AccessDenied');
+        expect(authErrorLog).toBeDefined();
+        expect(authErrorLog.length).toBe(1);
+    });
+
     it('should list objects by replication status', async () => {
         // Add data to source bucket
         log.info('Uploading test objects to source bucket');

tests/utils/S3Setup.js

@@ -1,5 +1,5 @@
 const { S3Client, CreateBucketCommand, ListObjectVersionsCommand, DeleteObjectsCommand, DeleteBucketCommand, PutBucketVersioningCommand, PutBucketReplicationCommand, DeleteBucketReplicationCommand, PutObjectCommand, HeadObjectCommand } = require('@aws-sdk/client-s3');
-const { IAMClient, CreateUserCommand, CreatePolicyCommand, CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, DeleteRoleCommand, DeletePolicyCommand, DeleteUserCommand } = require('@aws-sdk/client-iam');
+const { IAMClient, CreateUserCommand, CreatePolicyCommand, CreateRoleCommand, AttachRolePolicyCommand, DetachRolePolicyCommand, DeleteRoleCommand, DeletePolicyCommand, DeleteUserCommand, ListRolesCommand, ListAttachedRolePoliciesCommand, ListUsersCommand } = require('@aws-sdk/client-iam');
 const { promisify } = require('util');
 const { Logger } = require('werelogs');
 const admincredentials = require('vaultclient/tests/utils/admincredentials.json');
@@ -108,6 +108,55 @@ async function deleteTestAccount(vaultClient, account) {
     await account.s3Client.send(new DeleteBucketCommand({ Bucket: account.bucketName }));
     log.info('Deleted bucket', { bucket: account.bucketName });
 
+    // List and delete all IAM users
+    try {
+        const listUsersResp = await account.iamClient.send(new ListUsersCommand({}));
+        if (listUsersResp.Users && Array.isArray(listUsersResp.Users)) {
+            for (const user of listUsersResp.Users) {
+                await account.iamClient.send(new DeleteUserCommand({ UserName: user.UserName }));
+                log.info('Deleted IAM user', { iamUser: user.UserName });
+            }
+        }
+    } catch (err) {
+        log.error('Error listing or deleting IAM users', { error: err });
+    }
+
+    // List and delete all IAM roles owned by the test account
+    try {
+        const rolesResp = await account.iamClient.send(new ListRolesCommand({}));
+
+        if (rolesResp.Roles && Array.isArray(rolesResp.Roles)) {
+            for (const role of rolesResp.Roles) {
+                log.info('Deleting IAM role', { RoleName: role.RoleName });
+                try {
+                    // Before deleting, need to detach all policies from the role
+                    const attachedPolicies = await account.iamClient.send(
+                        new ListAttachedRolePoliciesCommand({ RoleName: role.RoleName })
+                    );
+                    if (attachedPolicies.AttachedPolicies) {
+                        for (const policy of attachedPolicies.AttachedPolicies) {
+                            await account.iamClient.send(
+                                new DetachRolePolicyCommand({
+                                    RoleName: role.RoleName,
+                                    PolicyArn: policy.PolicyArn
+                                })
+                            );
+                            log.info('Detached policy from role', { RoleName: role.RoleName, PolicyArn: policy.PolicyArn });
+                        }
+                    }
+                    await account.iamClient.send(
+                        new DeleteRoleCommand({ RoleName: role.RoleName })
+                    );
+                    log.info('Deleted role', { RoleName: role.RoleName });
+                } catch (roleErr) {
+                    log.error('Error deleting IAM role', { RoleName: role.RoleName, error: roleErr });
+                }
+            }
+        }
+    } catch (err) {
+        log.error('Error listing IAM roles', { error: err });
+    }
+
     // Delete account with vaultclient
     await promisify(vaultClient.deleteAccount.bind(vaultClient))(account.accountName);
     log.info('Deleted account', { account: account.accountName });