diff --git a/lib/cli.js b/lib/cli.js index c1b58d96..5793fbfd 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -752,6 +752,34 @@ program }) }) +program + .command('storage-s3') + .description('start s3 storage') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('--bucket ' + , 'your s3 bucket name' + , String) + .option('--profile ' + , 'your aws credentials profile name' + , String + , 'stf-storage') + .option('--endpoint ' + , 'your buckets endpoint' + , String + , 's3-ap-northeast-1.amazonaws.com') + .action(function(options) { + require('./units/storage/amazons3')({ + port: options.port + , profile: options.profile + , bucket: options.bucket + , endpoint: options.endpoint + , expires: options.expires + }) + }) + program .command('storage-plugin-image') .description('start storage image plugin') @@ -916,10 +944,18 @@ program , 'websocket port' , Number , 7110) + .option('--storage-type ' + , 'storage type' + , String + , 'temp') .option('--storage-port ' , 'storage port' , Number , 7102) + .option('--storage-options ' + , 'array of options to pass to the storage implementation' + , String + , '[]') .option('--storage-plugin-image-port ' , 'storage image plugin port' , Number @@ -1086,9 +1122,9 @@ program // storage , procutil.fork(__filename, [ - 'storage-temp' + util.format('storage-%s', options.storageType) , '--port', options.storagePort - ]) + ].concat(JSON.parse(options.storageOptions))) // image processor , procutil.fork(__filename, [ diff --git a/lib/units/storage/amazons3.js b/lib/units/storage/amazons3.js new file mode 100644 index 00000000..1bdae6da --- /dev/null +++ b/lib/units/storage/amazons3.js @@ -0,0 +1,149 @@ +var http = require('http') +var util = require('util') +var path = require('path') +var fs = require('fs') + +var express = require('express') +var validator = require('express-validator') +var bodyParser = require('body-parser') +var formidable = require('formidable') +var Promise = require('bluebird') +var uuid = require('node-uuid') +var AWS = require('aws-sdk') + +var lifecycle = require('../../util/lifecycle') +var logger = require('../../util/logger') +var requtil = require('../../util/requtil') + + +module.exports = function(options) { + var log = logger.createLogger('storage:s3') + , app = express() + , server = http.createServer(app) + , credentials = new AWS.SharedIniFileCredentials({ + profile: options.profile + }) + + AWS.config.credentials = credentials; + var s3 = new AWS.S3(options) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.use(bodyParser.json()) + app.use(validator()) + + function putObject(plugin, file) { + return new Promise(function(resolve, reject) { + var id = uuid.v4() + var rs = fs.createReadStream(file.path) + + s3.putObject({ + Key: id + , Body: rs + , Bucket: options.bucket + , Metadata: { + plugin: plugin + , name: file.name + } + }, function(err, data) { + if (err) { + log.error('failed to store "%s" bucket:"%s"', id, options.bucket) + log.error(err); + reject(err); + } else { + log.info('Stored "%s" to %s/%s', file.name, options.bucket, id) + resolve(id) + } + }) + }) + } + + function getHref(plugin, id, name) { + return util.format( + '/s/%s/%s%s' + , plugin + , id + , name ? '/' + path.basename(name) : '' + ) + } + + app.post('/s/upload/:plugin', function(req, res, next) { + var form = new formidable.IncomingForm() + var plugin = req.params.plugin + Promise.promisify(form.parse, form)(req) + .spread(function(fields, files) { + var requests = Object.keys(files).map(function(field) { + var file = files[field] + log.info('Uploaded "%s" to "%s"', file.name, file.path) + return putObject(plugin, file) + .then(function (id) { + return { + field: field + , id: id + , name: file.name + , temppath: file.path + } + }) + }) + return Promise.all(requests) + }) + .then(function(storedFiles) { + res.status(201).json({ + success: true, + resources: (function() { + var mapped = Object.create(null) + storedFiles.forEach(function(file) { + var plugin = req.params.plugin + mapped[file.field] = { + date: new Date() + , plugin: plugin + , id: file.id + , name: file.name + , href: getHref(plugin, file.id, file.name) + } + }) + return mapped + })() + }) + return storedFiles + }) + .then(function (storedFiles){ + storedFiles.forEach(function (file){ + fs.unlink(file.temppath) + }) + }) + .catch(function(err) { + log.error('Error storing resource', err.stack) + res.status(500) + .json({ + success: false, + error: 'ServerError' + }) + }) + }) + + app.get('/s/blob/:id/:name', function(req, res) { + var params = { + Key: req.params.id, + Bucket: options.bucket + } + s3.getObject(params, function(err, data) { + if (err) { + log.error('failed to retreive[' + path + ']') + log.error(err, err.stack); + res.sendStatus(404) + } else { + res.set({ + 'Content-type': data.ContentType + }) + res.send(data.Body) + } + }) + }) + + // initialize + server.listen(options.port) + console.log('Listening on port %d', options.port) +} diff --git a/package.json b/package.json index 84b23421..913b9d61 100644 --- a/package.json +++ b/package.json @@ -33,6 +33,7 @@ "adbkit": "^2.3.1", "adbkit-apkreader": "^1.0.0", "adbkit-monkey": "^1.0.1", + "aws-sdk": "^2.2.3", "bluebird": "^2.9.34", "body-parser": "^1.13.3", "chalk": "~1.1.1",