diff --git a/src/services/fileStorage/proxy-service.js b/src/services/fileStorage/proxy-service.js index 566ec8ccf0b..7d742e20556 100644 --- a/src/services/fileStorage/proxy-service.js +++ b/src/services/fileStorage/proxy-service.js @@ -34,7 +34,6 @@ const { equal: equalIds } = require('../../helper/compare').ObjectId; const { FILE_PREVIEW_SERVICE_URI, FILE_PREVIEW_CALLBACK_URI, - ENABLE_THUMBNAIL_GENERATION, FILE_SECURITY_CHECK_MAX_FILE_SIZE, SECURITY_CHECK_SERVICE_PATH, } = require('../../../config/globals'); @@ -44,40 +43,94 @@ const sanitizeObj = (obj) => { return obj; }; -const prepareThumbnailGeneration = (file, strategy, userId, { name: dataName }, { storageFileName, name: propName }) => - ENABLE_THUMBNAIL_GENERATION - ? Promise.all([ - strategy.getSignedUrl({ - userId, - flatFileName: storageFileName, - localFileName: storageFileName, - download: true, - Expires: 3600 * 24, - }), - strategy.generateSignedUrl({ - userId, - flatFileName: storageFileName.replace(/(\..+)$/, '-thumbnail.png'), - fileType: returnFileType(dataName || propName), // data.type - }), - ]).then(([downloadUrl, signedS3Url]) => - rp - .post({ - url: FILE_PREVIEW_SERVICE_URI, - body: { - downloadUrl, - signedS3Url, - callbackUrl: url.resolve(FILE_PREVIEW_CALLBACK_URI, file.thumbnailRequestToken), - options: { - width: 120, - }, +const getStorageProviderIdAndBucket = async (userId, fileObject, strategy) => { + let storageProviderId = fileObject.storageProviderId; + let bucket = fileObject.bucket; + + if (!storageProviderId) { + // deprecated: author check via file.permissions[0]?.refId is deprecated and will be removed in the next release + const creatorId = + fileObject.creator || + (fileObject.permissions[0]?.refPermModel !== 'user' ? userId : fileObject.permissions[0]?.refId); + + const creator = await userModel.findById(creatorId).exec(); + if (!creator || !creator.schoolId) { + throw new NotFound('User not found'); + } + + const { schoolId } = creator; + + const school = await schoolModel + .findOne({ _id: schoolId }, null, { readPreference: 'primary' }) // primary for afterhook in school.create + .populate('storageProvider') + .select(['storageProvider']) + .lean() + .exec(); + if (school === null) { + throw new NotFound('School not found.'); + } + + storageProviderId = school.storageProvider; + bucket = strategy.getBucket(schoolId); + } + + return { + storageProviderId, + bucket, + }; +}; + +const prepareThumbnailGeneration = async ( + file, + strategy, + userId, + { name: dataName }, + { storageFileName, name: propName } +) => { + if (Configuration.get('ENABLE_THUMBNAIL_GENERATION') === true) { + const fileObject = await FileModel.findOne({ _id: file }).lean().exec(); + + if (!fileObject) { + throw new NotFound('File seems not to be there.'); + } + + const { storageProviderId, bucket } = await getStorageProviderIdAndBucket(userId, fileObject); + + Promise.all([ + strategy.getSignedUrl({ + storageProviderId, + bucket, + flatFileName: storageFileName, + localFileName: storageFileName, + download: true, + Expires: 3600 * 24, + }), + strategy.generateSignedUrl({ + userId, + flatFileName: storageFileName.replace(/(\..+)$/, '-thumbnail.png'), + fileType: returnFileType(dataName || propName), // data.type + }), + ]).then(([downloadUrl, signedS3Url]) => + rp + .post({ + url: FILE_PREVIEW_SERVICE_URI, + body: { + downloadUrl, + signedS3Url, + callbackUrl: url.resolve(FILE_PREVIEW_CALLBACK_URI, file.thumbnailRequestToken), + options: { + width: 120, }, - json: true, - }) - .catch((err) => { - logger.warning(new Error('Can not create tumbnail', err)); // todo err message is lost and throw error - }) - ) - : Promise.resolve(); + }, + json: true, + }) + .catch((err) => { + logger.warning(new Error('Can not create tumbnail', err)); // todo err message is lost and throw error + }) + ); + } + return Promise.resolve(); +}; /** * @@ -86,7 +139,7 @@ const prepareThumbnailGeneration = (file, strategy, userId, { name: dataName }, * @param {FileStorageStrategy} strategy the file storage strategy used * @returns {Promise} Promise that rejects with errors or resolves with no data otherwise */ -const prepareSecurityCheck = (file, userId, strategy) => { +const prepareSecurityCheck = async (file, userId, strategy) => { if (Configuration.get('ENABLE_FILE_SECURITY_CHECK') === true) { if (file.size > FILE_SECURITY_CHECK_MAX_FILE_SIZE) { return FileModel.updateOne( @@ -99,10 +152,12 @@ const prepareSecurityCheck = (file, userId, strategy) => { } ).exec(); } + const { storageProviderId, bucket } = await getStorageProviderIdAndBucket(userId, file); // create a temporary signed URL and provide it to the virus scan service return strategy .getSignedUrl({ - userId, + storageProviderId, + bucket, flatFileName: file.storageFileName, localFileName: file.storageFileName, download: true, @@ -422,10 +477,7 @@ const signedUrlService = { throw new NotFound('File seems not to be there.'); } - // deprecated: author check via file.permissions[0]?.refId is deprecated and will be removed in the next release - const creatorId = - fileObject.creator || - (fileObject.permissions[0]?.refPermModel !== 'user' ? userId : fileObject.permissions[0]?.refId); + const { storageProviderId, bucket } = await getStorageProviderIdAndBucket(userId, fileObject); if (download && fileObject.securityCheck && fileObject.securityCheck.status === SecurityCheckStatusTypes.BLOCKED) { throw new Forbidden('File access blocked by security check.'); @@ -434,11 +486,11 @@ const signedUrlService = { return canRead(userId, file) .then(() => strategy.getSignedUrl({ - userId: creatorId, + storageProviderId, + bucket, flatFileName: fileObject.storageFileName, localFileName: query.name || fileObject.name, download: true, - bucket: fileObject.bucket, }) ) .then((res) => ({ @@ -457,16 +509,13 @@ const signedUrlService = { throw new NotFound('File seems not to be there.'); } - // deprecated: author check via file.permissions[0]?.refId is deprecated and will be removed in the next release - const creatorId = - fileObject.creator || fileObject.permissions[0]?.refPermModel !== 'user' - ? userId - : fileObject.permissions[0]?.refId; + const { storageProviderId, bucket } = await getStorageProviderIdAndBucket(userId, fileObject); return canRead(userId, id) .then(() => strategy.getSignedUrl({ - userId: creatorId, + storageProviderId, + bucket, flatFileName: fileObject.storageFileName, action: 'putObject', }) diff --git a/src/services/fileStorage/strategies/awsS3.js b/src/services/fileStorage/strategies/awsS3.js index c618d549749..d9994cf0faa 100644 --- a/src/services/fileStorage/strategies/awsS3.js +++ b/src/services/fileStorage/strategies/awsS3.js @@ -119,7 +119,7 @@ const listBuckets = async (awsObject) => { const getBucketName = (schoolId) => `${BUCKET_NAME_PREFIX}${schoolId}`; -const createAWSObject = async (schoolId) => { +const createAWSObjectFromSchoolId = async (schoolId) => { const school = await schoolModel .findOne({ _id: schoolId }, null, { readPreference: 'primary' }) // primary for afterhook in school.create .populate('storageProvider') @@ -152,6 +152,33 @@ const createAWSObject = async (schoolId) => { // end legacy }; +const createAWSObjectFromStorageProviderIdAndBucket = async (storageProviderId, bucket) => { + if (Configuration.get('FEATURE_MULTIPLE_S3_PROVIDERS_ENABLED') === true) { + const storageProvider = await StorageProviderModel.findOne({ _id: storageProviderId }).lean().exec(); + + if (!storageProvider) { + throw new NotFound('Storage provider not found.'); + } + + const s3 = getS3(storageProvider); + return { + s3, + bucket, + }; + } + + // begin legacy + if (!awsConfig.endpointUrl) throw new Error('S3 integration is not configured on the server'); + const config = new aws.Config(awsConfig); + config.endpoint = new aws.Endpoint(awsConfig.endpointUrl); + + return { + s3: new aws.S3(config), + bucket, + }; + // end legacy +}; + /** * split files-list in files, that are in current directory, and the sub-directories * @param data is the files-list @@ -305,7 +332,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { throw new BadRequest('No school id parameter given.'); } - const awsObject = await createAWSObject(schoolId); + const awsObject = await createAWSObjectFromSchoolId(schoolId); const data = await createBucket(awsObject); return { message: 'Successfully created s3-bucket!', @@ -358,7 +385,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { return new GeneralError('school not set'); } - return createAWSObject(result.schoolId).then((awsObject) => { + return createAWSObjectFromSchoolId(result.schoolId).then((awsObject) => { const params = { Bucket: awsObject.bucket, Prefix: path, @@ -384,7 +411,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { return new NotFound('User not found'); } - return createAWSObject(result.schoolId).then((awsObject) => { + return createAWSObjectFromSchoolId(result.schoolId).then((awsObject) => { // files can be copied to different schools const sourceBucket = `bucket-${externalSchoolId || result.schoolId}`; @@ -413,7 +440,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { if (!result || !result.schoolId) { return new NotFound('User not found'); } - return createAWSObject(result.schoolId).then((awsObject) => { + return createAWSObjectFromSchoolId(result.schoolId).then((awsObject) => { const params = { Bucket: awsObject.bucket, Delete: { @@ -444,7 +471,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { if (!result || !result.schoolId) { return new NotFound('User not found'); } - return createAWSObject(result.schoolId).then((awsObject) => + return createAWSObjectFromSchoolId(result.schoolId).then((awsObject) => this.createIfNotExists(awsObject).then((safeAwsObject) => { const params = { Bucket: safeAwsObject.bucket, @@ -462,33 +489,25 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { }); } - getSignedUrl({ userId, flatFileName, localFileName, download, action = 'getObject', bucket = undefined }) { - if (!userId || !flatFileName) { - return Promise.reject(new BadRequest('Missing parameters by getSignedUrl.', { userId, flatFileName })); + getSignedUrl({ storageProviderId, bucket, flatFileName, localFileName, download, action = 'getObject' }) { + if (!storageProviderId || !bucket || !flatFileName) { + return Promise.reject( + new BadRequest('Missing parameters by getSignedUrl.', { storageProviderId, bucket, flatFileName }) + ); } - return UserModel.userModel - .findById(userId) - .lean() - .exec() - .then((result) => { - if (!result || !result.schoolId) { - return new NotFound('User not found'); - } - - return createAWSObject(result.schoolId).then((awsObject) => { - const params = { - Bucket: bucket || awsObject.bucket, - Key: flatFileName, - Expires: Configuration.get('STORAGE_SIGNED_URL_EXPIRE'), - }; - const getBoolean = (value) => value === true || value === 'true'; - if (getBoolean(download)) { - params.ResponseContentDisposition = `attachment; filename = "${localFileName.replace('"', '')}"`; - } - return promisify(awsObject.s3.getSignedUrl.bind(awsObject.s3), awsObject.s3)(action, params); - }); - }); + return createAWSObjectFromStorageProviderIdAndBucket(storageProviderId, bucket).then((awsObject) => { + const params = { + Bucket: bucket, + Key: flatFileName, + Expires: Configuration.get('STORAGE_SIGNED_URL_EXPIRE'), + }; + const getBoolean = (value) => value === true || value === 'true'; + if (getBoolean(download)) { + params.ResponseContentDisposition = `attachment; filename = "${localFileName.replace('"', '')}"`; + } + return promisify(awsObject.s3.getSignedUrl.bind(awsObject.s3), awsObject.s3)(action, params); + }); } /** ** @DEPRECATED *** */ @@ -508,7 +527,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { return new NotFound('User not found'); } - return createAWSObject(result.schoolId).then((awsObject) => { + return createAWSObjectFromSchoolId(result.schoolId).then((awsObject) => { const fileStream = fs.createReadStream(pathUtil.join(__dirname, '..', 'resources', '.scfake')); const params = { Bucket: awsObject.bucket, @@ -539,7 +558,7 @@ class AWSS3Strategy extends AbstractFileStorageStrategy { if (!result || !result.schoolId) { return new NotFound('User not found'); } - return createAWSObject(result.schoolId).then((awsObject) => { + return createAWSObjectFromSchoolId(result.schoolId).then((awsObject) => { const params = { Bucket: awsObject.bucket, Prefix: removeLeadingSlash(path),