Skip to content

Cloudserver adaptation post arsenal refacto #6044

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
benzekrimaha wants to merge 13 commits into
base: development/9.3
Choose a base branch
from
+1,915 −1,435
Open

Cloudserver adaptation post arsenal refacto #6044

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
83dec6f
object tests updates post unused functions removal
benzekrimaha Jan 29, 2026
4047074
bucket tests updates post unused functions removal
benzekrimaha Jan 29, 2026
671e7c6
bump arsenal 8.3.0
benzekrimaha Jan 29, 2026
4572f8e
remove listBucketObjects from gcpUtils
benzekrimaha Jan 30, 2026
ff04567
async/await on objects tests
benzekrimaha Jan 30, 2026
b70996d
async/await on buckets tests
benzekrimaha Jan 30, 2026
6d272c9
fixup! remove listBucketObjects from gcpUtils
benzekrimaha Jan 30, 2026
5df1496
fixup! async/await on objects tests
benzekrimaha Jan 30, 2026
ebff875
fixup! async/await on objects tests
benzekrimaha Jan 30, 2026
d4d21a7
fixup! async/await on objects tests
benzekrimaha Jan 30, 2026
de486c3
fixup! async/await on objects tests
benzekrimaha Jan 30, 2026
af76373
fixup! async/await on objects tests
benzekrimaha Jan 30, 2026
fc5946f
fixup! async/await on objects tests
benzekrimaha Jan 30, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@
"@azure/storage-blob": "^12.28.0",
"@hapi/joi": "^17.1.1",
"@smithy/node-http-handler": "^3.0.0",
"arsenal": "git+https://github.com/scality/Arsenal#8.3.0-preview.1",
"arsenal": "git+https://github.com/scality/Arsenal#8.3.0",
"async": "2.6.4",
"bucketclient": "scality/bucketclient#8.2.7",
"bufferutil": "^4.0.8",
Expand Down
182 changes: 86 additions & 96 deletions tests/functional/raw-node/test/GCP/bucket/get.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
const assert = require('assert');
const async = require('async');
const arsenal = require('arsenal');
const {
ListObjectsCommand,
HeadBucketCommand,
CreateBucketCommand,
DeleteBucketCommand,
PutObjectCommand,
DeleteObjectCommand,
} = require('@aws-sdk/client-s3');
const { GCP } = arsenal.storage.data.external.GCP;
const { makeGcpRequest } = require('../../../utils/makeRequest');
const { gcpRequestRetry, genUniqID } = require('../../../utils/gcpUtils');
const { genUniqID, gcpRetry } = require('../../../utils/gcpUtils');
const { getRealAwsConfig } =
require('../../../../aws-node-sdk/test/support/awsConfig');
const { listingHardLimit } = require('../../../../../../constants');
Expand All @@ -18,95 +25,90 @@ const gcpClient = new GCP(config);
function populateBucket(createdObjects, callback) {
process.stdout.write(
`Putting ${createdObjects.length} objects into bucket\n`);
async.mapLimit(createdObjects, 10,
(object, moveOn) => {
makeGcpRequest({
method: 'PUT',
bucket: bucketName,
objectKey: object,
authCredentials: config.credentials,
}, err => {
moveOn(err);
});
}, err => {
if (err) {
process.stdout
.write(`err putting objects ${err.code}\n`);
async.mapLimit(
createdObjects,
10,
async object => {
const command = new PutObjectCommand({
Bucket: bucketName,
Key: object,
});
await gcpClient.send(command);
},
err => {
if (err) {
process.stdout
.write(`err putting objects ${err}\n`);
}
return callback(err);
}
return callback(err);
});
);
}

function removeObjects(createdObjects, callback) {
process.stdout.write(
`Deleting ${createdObjects.length} objects from bucket\n`);
async.mapLimit(createdObjects, 10,
(object, moveOn) => {
makeGcpRequest({
method: 'DELETE',
bucket: bucketName,
objectKey: object,
authCredentials: config.credentials,
}, err => moveOn(err));
}, err => {
if (err) {
process.stdout
.write(`err deleting objects ${err.code}\n`);
async.mapLimit(
createdObjects,
10,
async object => {
const command = new DeleteObjectCommand({
Bucket: bucketName,
Key: object,
});
await gcpClient.send(command);
},
err => {
if (err) {
process.stdout
.write(`err deleting objects ${err}\n`);
}
return callback(err);
}
return callback(err);
});
);
}

describe('GCP: GET Bucket', function testSuite() {
this.timeout(180000);

before(done => {
gcpRequestRetry({
method: 'PUT',
bucket: bucketName,
authCredentials: config.credentials,
}, 0, err => {
if (err) {
process.stdout.write(`err in creating bucket ${err}\n`);
}
return done(err);
});
before(async () => {
await gcpRetry(
gcpClient,
new CreateBucketCommand({ Bucket: bucketName }),
);
});

after(done => {
gcpRequestRetry({
method: 'DELETE',
bucket: bucketName,
authCredentials: config.credentials,
}, 0, err => {
if (err) {
process.stdout.write(`err in deleting bucket ${err}\n`);
}
return done(err);
});
after(async () => {
await gcpRetry(
gcpClient,
new DeleteBucketCommand({ Bucket: bucketName }),
);
});

describe('without existing bucket', () => {
it('should return 404 and NoSuchBucket', done => {
it('should return 404 and NoSuchBucket', async () => {
const badBucketName = `nonexistingbucket-${genUniqID()}`;
gcpClient.getBucket({
Bucket: badBucketName,
}, err => {
try {
const command = new HeadBucketCommand({
Bucket: badBucketName,
});
await gcpClient.send(command);
assert.fail('Expected NoSuchBucket error, but got success');
} catch (err) {
assert(err);
assert.strictEqual(err.$metadata?.httpStatusCode, 404);
assert.strictEqual(err.name, 'NoSuchBucket');
return done();
});
const statusCode = err.$metadata && err.$metadata.httpStatusCode;
assert.strictEqual(statusCode, 404);
const errorName = err.name === 'NotFound' ? 'NoSuchBucket' : err.name;
assert.strictEqual(errorName, 'NoSuchBucket');
}
});

it('should return 200', done => {
gcpClient.listObjects({
it('should return 200', async () => {
const command = new ListObjectsCommand({
Bucket: bucketName,
}, (err, res) => {
assert.equal(err, null, `Expected success, but got ${err}`);
assert.strictEqual(res.$metadata?.httpStatusCode, 200);
return done();
});
const res = await gcpClient.send(command);
assert.strictEqual(res.$metadata?.httpStatusCode, 200);
});
});

Expand All @@ -119,27 +121,22 @@ describe('GCP: GET Bucket', function testSuite() {

after(done => removeObjects(createdObjects, done));

it(`should list all ${smallSize} created objects`, done => {
gcpClient.listObjects({
it(`should list all ${smallSize} created objects`, async () => {
const command = new ListObjectsCommand({
Bucket: bucketName,
}, (err, res) => {
assert.equal(err, null, `Expected success, but got ${err}`);
assert.strictEqual(res.Contents.length, smallSize);
return done();
});
const res = await gcpClient.send(command);
assert.strictEqual(res.Contents.length, smallSize);
});

describe('with MaxKeys at 10', () => {
it('should list MaxKeys number of objects', done => {
gcpClient.listObjects({
it('should list MaxKeys number of objects', async () => {
const command = new ListObjectsCommand({
Bucket: bucketName,
MaxKeys: 10,
}, (err, res) => {
assert.equal(err, null,
`Expected success, but got ${err}`);
assert.strictEqual(res.Contents.length, 10);
return done();
});
const res = await gcpClient.send(command);
assert.strictEqual(res.Contents.length, 10);
});
});
});
Expand All @@ -152,15 +149,12 @@ describe('GCP: GET Bucket', function testSuite() {

after(done => removeObjects(createdObjects, done));

it('should list at max 1000 of objects created', done => {
gcpClient.listObjects({
it('should list at max 1000 of objects created', async () => {
const command = new ListObjectsCommand({
Bucket: bucketName,
}, (err, res) => {
assert.equal(err, null, `Expected success, but got ${err}`);
assert.strictEqual(res.Contents.length,
listingHardLimit);
return done();
});
const res = await gcpClient.send(command);
assert.strictEqual(res.Contents.length, listingHardLimit);
});

describe('with MaxKeys at 1001', () => {
Expand All @@ -175,17 +169,13 @@ describe('GCP: GET Bucket', function testSuite() {
//
// Actual behavior: it returns a list longer than 1000 objects when
// max-keys is greater than 1000
it.skip('should list at max 1000, ignoring MaxKeys', done => {
gcpClient.listObjects({
it.skip('should list at max 1000, ignoring MaxKeys', async () => {
const command = new ListObjectsCommand({
Bucket: bucketName,
MaxKeys: 1001,
}, (err, res) => {
assert.equal(err, null,
`Expected success, but got ${err}`);
assert.strictEqual(res.Contents.length,
listingHardLimit);
return done();
});
const res = await gcpClient.send(command);
assert.strictEqual(res.Contents.length, listingHardLimit);
});
});
});
Expand Down
Loading
Loading