From 8b4851317597f76e22e1bb448dac07e904774743 Mon Sep 17 00:00:00 2001 From: Alexander Sadovsky Date: Thu, 9 Aug 2018 22:23:09 +0200 Subject: [PATCH] added support for assets/tasks file upload. closes #31 --- .gitignore | 3 +- api/controllers/AssetController.ts | 68 ++-- api/controllers/TaskController.ts | 64 ++-- api/models/Asset.ts | 11 +- api/models/Task.ts | 11 +- config/routes.js | 291 ++++++++++-------- test/fixtures/testAssetFile.txt | 1 + test/fixtures/testDeliverableFile.txt | 1 + .../controllers/AssetController.test.js | 54 ++++ .../controllers/TaskController.test.js | 62 ++++ yarn.lock | 4 +- 11 files changed, 376 insertions(+), 194 deletions(-) create mode 100644 test/fixtures/testAssetFile.txt create mode 100644 test/fixtures/testDeliverableFile.txt create mode 100644 test/integration/controllers/AssetController.test.js create mode 100644 test/integration/controllers/TaskController.test.js diff --git a/.gitignore b/.gitignore index 09b9628..c390f7c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ typings - +assets/assets +assets/deliverables ################################################ diff --git a/api/controllers/AssetController.ts b/api/controllers/AssetController.ts index b96a973..65d3b96 100644 --- a/api/controllers/AssetController.ts +++ b/api/controllers/AssetController.ts @@ -4,40 +4,50 @@ * @description :: Server-side logic for managing assets * @help :: See http://sailsjs.org/#!/documentation/concepts/Controllers */ -import express = require('express'); import { Model } from 'sails-typings'; declare var Asset: Model; -declare var Job: Model; - +declare var sails; +const path = require('path'); module.exports = { - 'uploadFile': (req: express.Request, res: express.Response) => { - - // request body should contain: - // isReference - // sourceLanguage - // encoding - // tasks - - const id = req.param('id'); - console.log(id); - - Job.find({ id }).exec((err, theUser) => { - console.log(err); - console.log(theUser); - if (err) return res.send(err); - if (!theUser || theUser.length < 1) return res.send(`job ${id} not found`); - console.log('test'); - }); - - - return res.send('body:' + JSON.stringify(req.body)); - }, - - 'downloadFile': (req, res) => { + 'uploadFile': (req, res) => { + const dirname = path.resolve(sails.config.appPath, 'assets/assets'); + req.file('asset').upload({ dirname }, function (err, uploadedFiles) { + if (err) return res.serverError(err); + Asset.create({ + sourceLanguage: req.param('sourceLanguage'), + encoding: req.param('encoding'), + jobId: req.param('parentid'), + fileDescriptor: uploadedFiles[0].fd, + fileOriginalName: uploadedFiles[0].filename + }, function (err, result) { + if (err) return res.serverError(err); + return res.json(result); + }); + }); + }, + + 'downloadFile': (req, res) => { + Asset.findOne(req.param('id')).exec(function (err, asset) { + if (err) return res.serverError(err); + if (!asset) return res.notFound(); + if (!asset.fileDescriptor) return res.notFound(); + + const SkipperDisk = require('skipper-disk'); + const fileAdapter = SkipperDisk(); + + // set the filename to the same file as the user uploaded + res.set("Content-disposition", "attachment; filename=" + asset.fileOriginalName + ""); + + // Stream the file down + fileAdapter.read(asset.fileDescriptor) + .on('error', function (err) { + return res.serverError(err); + }) + .pipe(res); + }); + } - } - }; diff --git a/api/controllers/TaskController.ts b/api/controllers/TaskController.ts index 10ab54f..3adb933 100644 --- a/api/controllers/TaskController.ts +++ b/api/controllers/TaskController.ts @@ -4,40 +4,50 @@ * @description :: Server-side logic for managing tasks * @help :: See http://sailsjs.org/#!/documentation/concepts/Controllers */ -import express = require('express'); import { Model } from 'sails-typings'; declare var Task: Model; -declare var Job: Model; +declare var sails; +const path = require('path'); + module.exports = { - 'uploadFile': (req: express.Request, res: express.Response) => { - - // request body should contain: - // isReference - // sourceLanguage - // encoding - // tasks - - const id = req.param('id'); - console.log(id); - - Job.find({ id }).exec((err, theUser) => { - console.log(err); - console.log(theUser); - if (err) return res.send(err); - if (!theUser || theUser.length < 1) return res.send(`job ${id} not found`); - console.log('test'); + 'uploadFile': (req, res) => { + const dirname = path.resolve(sails.config.appPath, 'assets/deliverables'); + req.file('deliverable').upload({ dirname }, function (err, uploadedFiles) { + if (err) return res.serverError(err); + Task.update(req.param('id'), { + progress: 'finished', + fileDescriptor: uploadedFiles[0].fd, + fileOriginalName: uploadedFiles[0].filename + }, function (err, result) { + if (err) return res.serverError(err); + return res.json(result); + }); }); - - - return res.send('body:' + JSON.stringify(req.body)); - }, - - 'downloadFile': (req, res) => { - - } + }, + + 'downloadFile': (req, res) => { + Task.findOne(req.param('id')).exec(function (err, task) { + if (err) return res.serverError(err); + if (!task) return res.notFound(); + if (!task.fileDescriptor) return res.notFound(); + + const SkipperDisk = require('skipper-disk'); + const fileAdapter = SkipperDisk(); + + // set the filename to the same file as the user uploaded + res.set("Content-disposition", "attachment; filename=" + task.fileOriginalName + ""); + + // Stream the file down + fileAdapter.read(task.fileDescriptor) + .on('error', function (err) { + return res.serverError(err); + }) + .pipe(res); + }); + } }; diff --git a/api/models/Asset.ts b/api/models/Asset.ts index bcfdc1a..3dd418c 100644 --- a/api/models/Asset.ts +++ b/api/models/Asset.ts @@ -16,9 +16,14 @@ module.exports = { description: '(auto-generated)' }, - file: { - type: 'binary', - description: 'an actual file (asset)' + fileDescriptor: { + type: 'string', + description: 'unique name of the file. (auto-generated)' + }, + + fileOriginalName: { + type: 'string', + description: 'original name of the file as uploaded. (auto-filled)' }, isReference: { diff --git a/api/models/Task.ts b/api/models/Task.ts index 213ba6e..536ad25 100644 --- a/api/models/Task.ts +++ b/api/models/Task.ts @@ -48,9 +48,14 @@ finished - the work on this Task is done and deliverableLocation is filled with example: 'symfonie.com/43920149320' }, - file: { - type: 'binary', - description: 'an actual file (deliverable)' + fileDescriptor: { + type: 'string', + description: 'unique name of the file. (auto-generated)' + }, + + fileOriginalName: { + type: 'string', + description: 'original name of the file as uploaded. (auto-filled)' }, jobId: { diff --git a/config/routes.js b/config/routes.js index 0017cac..0dfb46a 100644 --- a/config/routes.js +++ b/config/routes.js @@ -69,14 +69,14 @@ module.exports.routes = { }, 'POST /job/:id/submit': - { - controller: 'JobController', - action: 'create', - swagger: { - summary: 'Submit a Job', - description: 'Submit a Job, optionally with assets and tasks' - } - }, + { + controller: 'JobController', + action: 'create', + swagger: { + summary: 'Submit a Job', + description: 'Submit a Job, optionally with assets and tasks' + } + }, 'DELETE /job/:parentid/asset/:id': { controller: 'AssetController', @@ -220,157 +220,190 @@ module.exports.routes = { } }, - // todo -> make file upload and download work - 'POST /job/:parentid/asset/uploadfile': // todo -> this endpoint also creates an asset - { - controller: 'AssetController', - action: 'uploadFile', - swagger: { - summary: 'Upload an asset file and create an asset', - description: 'Upload an asset file and create an asset', - tags: [ - 'Asset' - ], - responses: { - '200': { - schema: { - "$ref": "#/definitions/asset", - } - } - } - } - }, - 'GET /job/:parentid/asset/:id/downloadfile': { + 'POST /job/:parentid/asset/uploadfile': + { controller: 'AssetController', - action: 'downloadFile', + action: 'uploadFile', swagger: { - summary: 'Download an asset file', - description: 'Download an asset file', - tags: [ - 'Asset' + summary: 'Upload an asset file and create an asset', + description: 'Upload an asset file and create an asset', + parameters: [ + { + "name": "parentid", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "sourceLanguage", + "in": "formData", + "type": "string" + }, + { + "name": "encoding", + "in": "formData", + "type": "string" + }, + { + "name": "asset", + "in": "formData", + "description": "The Asset file", + "required": true, + "type": "file" + }, ], responses: { '200': { schema: { - "$ref": "", - type: 'file' + "$ref": "#/definitions/asset", } } } } }, - 'POST /asset/:parentid/task/:id/uploaddeliverable': { - controller: 'TaskController', - action: 'uploadFile', + 'GET /job/:parentid/asset/:id/downloadfile': { + controller: 'AssetController', + action: 'downloadFile', swagger: { - summary: 'Upload deliverable file', - description: 'Upload deliverable file', + summary: 'Download an asset file', + description: 'Download an asset file', tags: [ - 'Task' + 'Asset' ], responses: { - '200': { + 200: { + description: "The asset file", schema: { - "$ref": "" + type: "file" } } } } }, - 'GET /asset/:parentid/task/:id/downloaddeliverable': { - controller: 'TaskController', - action: 'downloadFile', - swagger: { - summary: 'Download deliverable file', - description: 'Download deliverable file', - produces: [ - 'application/json' - ], - tags: [ - 'Task' - ], - responses: { - '200': { - description: 'Deliverable file', - schema: { - "$ref": "", - type: 'file' + 'POST /asset/:parentid/task/:id/uploaddeliverable': { + controller: 'TaskController', + action: 'uploadFile', + swagger: { + summary: 'Upload deliverable file', + description: 'Upload deliverable file', + tags: [ + 'Task' + ], + parameters: [ + { + "name": "parentid", + "in": "path", + "required": true, + "type": "string" + }, + { + "name": "id", + "in": "path", + "required": true, + "type": "string" }, - content: { - 'application/json': { - schema: { - type: 'file' - } + { + "name": "deliverable", + "in": "formData", + "description": "The deliverable file", + "required": true, + "type": "file" + }, + ], + responses: { + '200': { + schema: { + "$ref": "#/definitions/task", } } } } + }, + 'GET /asset/:parentid/task/:id/downloaddeliverable': { + controller: 'TaskController', + action: 'downloadFile', + swagger: { + summary: 'Download deliverable file', + description: 'Download deliverable file', + produces: [ + 'application/json' + ], + tags: [ + 'Task' + ], + responses: { + 200: { + description: "The deliverable file", + schema: { + type: "file" + } + } + } + } + }, - } - }, - - // task - 'GET /task': { - controller: 'TaskController', - action: 'find', - swagger: { - summary: 'List all Tasks', - description: 'List all Tasks' - } - }, + // task + 'GET /task': { + controller: 'TaskController', + action: 'find', + swagger: { + summary: 'List all Tasks', + description: 'List all Tasks' + } + }, - // asset - 'GET /asset': { - controller: 'AssetController', - action: 'find', - swagger: { - summary: 'List all Assets', - description: 'List all Assets' - } - }, + // asset + 'GET /asset': { + controller: 'AssetController', + action: 'find', + swagger: { + summary: 'List all Assets', + description: 'List all Assets' + } + }, - // webhook - 'GET /webhook': { - controller: 'WebhookController', - action: 'find', - swagger: { - summary: 'List all Webhooks', - description: 'List all Webhooks' - } - }, - 'POST /webhook': { - controller: 'WebhookController', - action: 'create', - swagger: { - summary: 'Create a Webhook', - description: 'Create a Webhook' - } - }, + // webhook + 'GET /webhook': { + controller: 'WebhookController', + action: 'find', + swagger: { + summary: 'List all Webhooks', + description: 'List all Webhooks' + } + }, + 'POST /webhook': { + controller: 'WebhookController', + action: 'create', + swagger: { + summary: 'Create a Webhook', + description: 'Create a Webhook' + } + }, - 'DELETE /webhook/:id': { - controller: 'WebhookController', - action: 'destroy', - swagger: { - summary: 'Delete a Webhook', - description: 'Delete a Webhook' - } - }, - 'GET /webhook/:id': { - controller: 'WebhookController', - action: 'findOne', - swagger: { - summary: 'Get a Webhook', - description: 'Get a Webhook' - } - }, - 'PUT /webhook/:id': { - controller: 'WebhookController', - action: 'update', - swagger: { - summary: 'Update a Webhook', - description: 'Update a Webhook' + 'DELETE /webhook/:id': { + controller: 'WebhookController', + action: 'destroy', + swagger: { + summary: 'Delete a Webhook', + description: 'Delete a Webhook' + } + }, + 'GET /webhook/:id': { + controller: 'WebhookController', + action: 'findOne', + swagger: { + summary: 'Get a Webhook', + description: 'Get a Webhook' + } + }, + 'PUT /webhook/:id': { + controller: 'WebhookController', + action: 'update', + swagger: { + summary: 'Update a Webhook', + description: 'Update a Webhook' + } } - } -}; + }; diff --git a/test/fixtures/testAssetFile.txt b/test/fixtures/testAssetFile.txt new file mode 100644 index 0000000..3d8f2b4 --- /dev/null +++ b/test/fixtures/testAssetFile.txt @@ -0,0 +1 @@ +This is a fake file which "should" be translated to other language. \ No newline at end of file diff --git a/test/fixtures/testDeliverableFile.txt b/test/fixtures/testDeliverableFile.txt new file mode 100644 index 0000000..b66eb73 --- /dev/null +++ b/test/fixtures/testDeliverableFile.txt @@ -0,0 +1 @@ +Toto je falošný súbor (výsledok splnenej úlohy), ktorý je preložený do Slovenčiny. \ No newline at end of file diff --git a/test/integration/controllers/AssetController.test.js b/test/integration/controllers/AssetController.test.js new file mode 100644 index 0000000..5ee88b2 --- /dev/null +++ b/test/integration/controllers/AssetController.test.js @@ -0,0 +1,54 @@ +var request = require('supertest'); +var expect = require('expect'); +var _ = require('lodash'); + +describe('AssetController', function () { + + describe('POST /job/:parentid/asset/uploadfile', function () { + beforeEach(function (done) { + Job.create({ + id: 1, + name: 'first job', + submitter: 'symfonie.com/123' + }).exec((err, res) => { + if (err) console.error(err); + done(); + }); + }); + + it('should handle file upload and asset creation', function (done) { + request(sails.hooks.http.app) + .post('/job/1/asset/uploadfile') + .field('sourceLanguage', 'en') + .field('encoding', 'utf8') + .attach('asset', 'test/fixtures/testAssetFile.txt') + .expect(200) + .then((response) => { + expect(_.omit(response.body, ['updatedAt', 'createdAt'])).toEqual({ + id: 1, + jobId: 1, + sourceLanguage: 'en', + encoding: 'utf8', + fileDescriptor: response.body.fileDescriptor, + fileOriginalName: 'testAssetFile.txt' + }); + + Job.findOne(1).populate('assets').exec(function (err, res) { + expect(err).toBe(null) + expect(_.omit(res.toJSON(), ['createdAt', 'updatedAt'])).toEqual({ + id: 1, + name: 'first job', + submitter: 'symfonie.com/123', + assets: [response.body] + }); + done() + }); + }).catch(err => { + expect(err).not.toBeDefined() + done() + }); + }); + + }); + +}); \ No newline at end of file diff --git a/test/integration/controllers/TaskController.test.js b/test/integration/controllers/TaskController.test.js new file mode 100644 index 0000000..be292e8 --- /dev/null +++ b/test/integration/controllers/TaskController.test.js @@ -0,0 +1,62 @@ +var request = require('supertest'); +var expect = require('expect'); +var _ = require('lodash'); + +describe('TaskController', function () { + const fixtures = { + jobs: [{ + id: 1, + name: 'first job', + submitter: 'symfonie.com/123' + }], + assets: [{ + id: 1, + jobId: 1 + }], + tasks: [{ + id: 1, + assetId: 1, + progress: 'pending', + type: 'translation' + }] + } + describe.only('POST /asset/:parentid/task/:id/uploaddeliverable', function () { + beforeEach(function (done) { + Job.create(fixtures.jobs[0]).then(() => { + return Asset.create(fixtures.assets[0]) + }).then(() => { + return Task.create(fixtures.tasks[0]) + }).then(() => { + done() + }) + }); + + it('should upload the deliverable file to an existing Task', function (done) { + + request(sails.hooks.http.app) + .post('/asset/1/task/1/uploaddeliverable') + .attach('deliverable', 'test/fixtures/testDeliverableFile.txt') + .expect(200) + .then((response) => { + Asset.findOne(fixtures.assets[0].id).populate('tasks').exec(function (err, res) { + const asset = res.toJSON(); + const expectedTask = _.merge(fixtures.tasks[0], _.pick(asset.tasks[0], ['createdAt', 'updatedAt'])); + expectedTask.fileDescriptor = asset.tasks[0].fileDescriptor; + expectedTask.fileOriginalName = 'testDeliverableFile.txt'; + expectedTask.progress = 'finished'; + expect(err).toBe(null) + expect(asset).toEqual({ + id: 1, + jobId: 1, + createdAt: asset.createdAt, + updatedAt: asset.updatedAt, + tasks: [expectedTask] + }); + done() + }); + }); + }); + + }); + +}); \ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 90c3a88..711e825 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2713,9 +2713,9 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" -"lodash-new@git://github.com/lodash/lodash.git#4.17.10": +"lodash-new@git://github.com/lodash/lodash#4.17.10": version "4.17.10" - resolved "git://github.com/lodash/lodash.git#67389a8c78975d97505fa15aa79bec6397749807" + resolved "git://github.com/lodash/lodash#67389a8c78975d97505fa15aa79bec6397749807" lodash._basecopy@^3.0.0: version "3.0.1"