Skip to content

Commit

Permalink
Merge pull request #335 from ashmeet-kandhari/optimize_put_dir
Browse files Browse the repository at this point in the history 
creating a single client to put folders in cloud
  • Loading branch information
@ricardo-devis-agullo
ricardo-devis-agullo authored Jun 4, 2024
2 parents 59915ab + 601f97f commit 8436f8c
Show file tree
Hide file tree
Showing 10 changed files with 84 additions and 51 deletions.
3 changes: 2 additions & 1 deletion packages/oc-azure-storage-adapter/__test__/azure.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ const validOptions = {
privateContainerName: 'privcon',
accountName: 'name',
accountKey: 'key',
path: '/'
path: '/',
componentsDir: 'components'
};

test('should expose the correct methods', () => {
Expand Down
3 changes: 2 additions & 1 deletion packages/oc-azure-storage-adapter/__test__/privateFilesExclusion.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ test('put directory recognizes server.js and .env to be private', async () => {
privateContainerName: 'privcon',
accountName: 'name',
accountKey: 'key',
path: '/'
path: '/',
componentsDir: 'components'
});

const mockResult = (await client.putDir('.', '.')) as Array<{
Expand Down
41 changes: 25 additions & 16 deletions packages/oc-azure-storage-adapter/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,16 +58,21 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
refreshInterval: conf.refreshInterval
});

let client: BlobServiceClient | undefined = undefined;

const getClient = () => {
const sharedKeyCredential = new StorageSharedKeyCredential(
conf.accountName,
conf.accountKey
);
const blobServiceClient = new BlobServiceClient(
`https://${conf.accountName}.blob.core.windows.net`,
sharedKeyCredential
);
return blobServiceClient;
if (!client) {
const sharedKeyCredential = new StorageSharedKeyCredential(
conf.accountName,
conf.accountKey
);
client = new BlobServiceClient(
`https://${conf.accountName}.blob.core.windows.net`,
sharedKeyCredential
);

}
return client;
};

const getFile = async (filePath: string, force = false) => {
Expand Down Expand Up @@ -158,6 +163,7 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
const paths = await getPaths(dirInput);
const packageJsonFile = path.join(dirInput, 'package.json');
const files = paths.files.filter(file => file !== packageJsonFile);
const client = getClient();

const filesResults = await Promise.all(
files.map((file: string) => {
Expand All @@ -170,7 +176,8 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
return putFile(
file,
url,
privateFilePatterns.some(r => r.test(relativeFile))
privateFilePatterns.some(r => r.test(relativeFile)),
client
);
})
);
Expand All @@ -179,7 +186,8 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
const packageJsonFileResult = await putFile(
packageJsonFile,
`${dirOutput}/package.json`.replace(/\\/g, '/'),
false
false,
client
);

return [...filesResults, packageJsonFileResult];
Expand All @@ -188,7 +196,8 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
const putFileContent = async (
fileContent: string | fs.ReadStream,
fileName: string,
isPrivate: boolean
isPrivate: boolean,
client: BlobServiceClient
) => {
const content =
typeof fileContent === 'string'
Expand All @@ -208,8 +217,8 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
if (fileInfo.gzip) {
blobHTTPHeaders.blobContentEncoding = 'gzip';
}

const containerClient = getClient().getContainerClient(containerName);
const localClient = client ? client : getClient();
const containerClient = localClient.getContainerClient(containerName);
const blockBlobClient = containerClient.getBlockBlobClient(fileName);

return blockBlobClient.uploadData(content, {
Expand All @@ -224,9 +233,9 @@ export default function azureAdapter(conf: AzureConfig): StorageAdapter {
return result;
};

const putFile = (filePath: string, fileName: string, isPrivate: boolean) => {
const putFile = (filePath: string, fileName: string, isPrivate: boolean, client: BlobServiceClient) => {
const stream = fs.createReadStream(filePath);
return putFileContent(stream, fileName, isPrivate);
return putFileContent(stream, fileName, isPrivate, client);
};

return {
Expand Down
3 changes: 2 additions & 1 deletion packages/oc-gs-storage-adapter/__test__/gs.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ global.Date.now = _Date.now;
const validOptions = {
bucket: 'test',
projectId: '12345',
path: '/'
path: '/',
componentsDir: 'components'
};

test('should expose the correct methods', () => {
Expand Down
3 changes: 2 additions & 1 deletion packages/oc-gs-storage-adapter/__test__/privateFilesExclusion.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ test('put directory recognizes server.js and .env to be private', async () => {
const options = {
bucket: 'test',
projectId: '12345',
path: 'somepath'
path: 'somepath',
componentsDir: 'components'
};
const client = gs(options);

Expand Down
20 changes: 13 additions & 7 deletions packages/oc-gs-storage-adapter/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter {
const paths = await getPaths(dirInput);
const packageJsonFile = path.join(dirInput, 'package.json');
const files = paths.files.filter(file => file !== packageJsonFile);
const client = getClient();

const filesResults = await Promise.all(
files.map((file: string) => {
Expand All @@ -155,7 +156,8 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter {
return putFile(
file,
url,
privateFilePatterns.some(r => r.test(relativeFile))
privateFilePatterns.some(r => r.test(relativeFile)),
client
);
})
);
Expand All @@ -164,7 +166,8 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter {
const packageJsonFileResult = await putFile(
packageJsonFile,
`${dirOutput}/package.json`.replace(/\\/g, '/'),
false
false,
client
);

return [...filesResults, packageJsonFileResult];
Expand All @@ -173,14 +176,15 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter {
const putFileContent = async (
fileContent: string,
fileName: string,
isPrivate: boolean
isPrivate: boolean,
client: Storage
) => {
const tmpobj = tmp.fileSync();

fs.writeFileSync(tmpobj.name, fileContent);

try {
const result = await putFile(tmpobj.name, fileName, isPrivate);
const result = await putFile(tmpobj.name, fileName, isPrivate, client);
return result;
} finally {
tmpobj.removeCallback();
Expand All @@ -190,7 +194,8 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter {
const putFile = async (
filePath: string,
fileName: string,
isPrivate: boolean
isPrivate: boolean,
client: Storage
) => {
const fileInfo = getFileInfo(fileName);
const obj: {
Expand Down Expand Up @@ -222,11 +227,12 @@ export default function gsAdapter(conf: GsConfig): StorageAdapter {
};
}

const localClient = client ? client : getClient();
try {
await getClient().bucket(bucketName).upload(filePath, options);
await localClient.bucket(bucketName).upload(filePath, options);

if (obj.ACL === 'public-read') {
await getClient().bucket(bucketName).file(fileName).makePublic();
await localClient.bucket(bucketName).file(fileName).makePublic();
}

return obj;
Expand Down
3 changes: 2 additions & 1 deletion packages/oc-riak-storage-adapter/__test__/s3.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ const validOptions = {
region: 'region-test',
key: 'test-key',
secret: 'test-secret',
path: '/'
path: '/',
componentsDir: 'components'
};

test('should expose the correct methods', () => {
Expand Down
3 changes: 2 additions & 1 deletion packages/oc-s3-storage-adapter/__test__/privateFilesExclusion.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ test('put directory recognizes server.js and .env to be private', async () => {
region: 'region-test',
key: 'test-key',
secret: 'test-secret',
path: '/'
path: '/',
componentsDir: 'components'
};

const client = s3(options);
Expand Down
50 changes: 30 additions & 20 deletions packages/oc-s3-storage-adapter/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,22 +95,27 @@ export default function s3Adapter(conf: S3Config): StorageAdapter {
requestHandler = new NodeHttpHandler(handlerOptions);
}

let client: S3 | undefined = undefined;

const getClient = () => {
const configOpts: S3ClientConfig = {
logger: conf.debug ? (console.log as any) : undefined,
tls: sslEnabled,
requestHandler,
endpoint: conf.endpoint,
region,
forcePathStyle: s3ForcePathStyle
}
if (accessKeyId && secretAccessKey) {
configOpts.credentials = {
accessKeyId,
secretAccessKey
};
if (!client) {
const configOpts: S3ClientConfig = {
logger: conf.debug ? (console as any) : undefined,
tls: sslEnabled,
requestHandler,
endpoint: conf.endpoint,
region,
forcePathStyle: s3ForcePathStyle
}
if (accessKeyId && secretAccessKey) {
configOpts.credentials = {
accessKeyId,
secretAccessKey
};
}
client = new S3(configOpts);
}
return new S3(configOpts);
return client;
};

const getFile = async (filePath: string, force = false) => {
Expand Down Expand Up @@ -197,6 +202,7 @@ export default function s3Adapter(conf: S3Config): StorageAdapter {
const paths = await getPaths(dirInput);
const packageJsonFile = path.join(dirInput, 'package.json');
const files = paths.files.filter(file => file !== packageJsonFile);
const client = getClient();

const filesResults = await Promise.all(
files.map((file: string) => {
Expand All @@ -209,7 +215,8 @@ export default function s3Adapter(conf: S3Config): StorageAdapter {
return putFile(
file,
url,
privateFilePatterns.some(r => r.test(relativeFile))
privateFilePatterns.some(r => r.test(relativeFile)),
client
);
})
);
Expand All @@ -218,7 +225,8 @@ export default function s3Adapter(conf: S3Config): StorageAdapter {
const packageJsonFileResult = await putFile(
packageJsonFile,
`${dirOutput}/package.json`.replace(/\\/g, '/'),
false
false,
client
);

return [...filesResults, packageJsonFileResult];
Expand All @@ -227,11 +235,13 @@ export default function s3Adapter(conf: S3Config): StorageAdapter {
const putFileContent = async (
fileContent: string | fs.ReadStream,
fileName: string,
isPrivate: boolean
isPrivate: boolean,
client: S3
) => {
const fileInfo = getFileInfo(fileName);
const localClient = client ? client : getClient();

return getClient().putObject({
return localClient.putObject({
Bucket: bucket,
Key: fileName,
Body: fileContent,
Expand All @@ -243,10 +253,10 @@ export default function s3Adapter(conf: S3Config): StorageAdapter {
});
};

const putFile = (filePath: string, fileName: string, isPrivate: boolean) => {
const putFile = (filePath: string, fileName: string, isPrivate: boolean, client: S3) => {
const stream = fs.createReadStream(filePath);

return putFileContent(stream, fileName, isPrivate);
return putFileContent(stream, fileName, isPrivate, client);
};

return {
Expand Down
6 changes: 4 additions & 2 deletions packages/oc-storage-adapters-utils/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,14 @@ export interface StorageAdapter {
putFile(
filePath: string,
fileName: string,
isPrivate: boolean
isPrivate: boolean,
client?: unknown
): Promise<unknown>;
putFileContent(
data: unknown,
path: string,
isPrivate: boolean
isPrivate: boolean,
client?: unknown
): Promise<unknown>;
isValid: () => boolean;
}

0 comments on commit 8436f8c

Please sign in to comment.