Merge branch 's3insteadoftmp' of https://github.com/hogehoge-banana/stf into hogehoge-banana-s3insteadoftmp

lib/cli.js

@@ -752,6 +752,34 @@ program
     })
   })
 
+program
+  .command('storage-s3')
+  .description('start s3 storage')
+  .option('-p, --port <port>'
+    , 'port (or $PORT)'
+    , Number
+    , process.env.PORT || 7100)
+  .option('--bucket <bucketname>'
+    , 'your s3 bucket name'
+    , String)
+  .option('--profile <name>'
+    , 'your aws credentials profile name'
+    , String
+    , 'stf-storage')
+  .option('--endpoint <endpoint>'
+    , 'your buckets endpoint'
+    , String
+    , 's3-ap-northeast-1.amazonaws.com')
+  .action(function(options) {
+    require('./units/storage/amazons3')({
+      port: options.port
+    , profile: options.profile
+    , bucket: options.bucket
+    , endpoint: options.endpoint
+    , expires: options.expires
+    })
+  })
+
 program
   .command('storage-plugin-image')
   .description('start storage image plugin')
@@ -916,10 +944,18 @@ program
     , 'websocket port'
     , Number
     , 7110)
+  .option('--storage-type <temp|s3>'
+    , 'storage type'
+    , String
+    , 'temp')
   .option('--storage-port <port>'
     , 'storage port'
     , Number
     , 7102)
+  .option('--storage-options <json>'
+    , 'array of options to pass to the storage implementation'
+    , String
+    , '[]')
   .option('--storage-plugin-image-port <port>'
     , 'storage image plugin port'
     , Number
@@ -1086,9 +1122,9 @@ program
 
         // storage
       , procutil.fork(__filename, [
-            'storage-temp'
+            util.format('storage-%s', options.storageType)
           , '--port', options.storagePort
-          ])
+          ].concat(JSON.parse(options.storageOptions)))
 
         // image processor
       , procutil.fork(__filename, [

lib/units/storage/amazons3.js

@@ -0,0 +1,149 @@
+var http = require('http')
+var util = require('util')
+var path = require('path')
+var fs = require('fs')
+
+var express = require('express')
+var validator = require('express-validator')
+var bodyParser = require('body-parser')
+var formidable = require('formidable')
+var Promise = require('bluebird')
+var uuid = require('node-uuid')
+var AWS = require('aws-sdk')
+
+var lifecycle = require('../../util/lifecycle')
+var logger = require('../../util/logger')
+var requtil = require('../../util/requtil')
+
+
+module.exports = function(options) {
+  var log = logger.createLogger('storage:s3')
+  , app = express()
+  , server = http.createServer(app)
+  , credentials = new AWS.SharedIniFileCredentials({
+      profile: options.profile
+    })
+
+  AWS.config.credentials = credentials;
+  var s3 = new AWS.S3(options)
+
+  app.set('strict routing', true)
+  app.set('case sensitive routing', true)
+  app.set('trust proxy', true)
+
+  app.use(bodyParser.json())
+  app.use(validator())
+
+  function putObject(plugin, file) {
+    return new Promise(function(resolve, reject) {
+      var id = uuid.v4()
+      var rs = fs.createReadStream(file.path)
+
+      s3.putObject({
+        Key: id
+      , Body: rs
+      , Bucket: options.bucket
+      , Metadata: {
+          plugin: plugin
+        , name: file.name
+        }
+      }, function(err, data) {
+        if (err) {
+          log.error('failed to store "%s" bucket:"%s"', id, options.bucket)
+          log.error(err);
+          reject(err);
+        } else {
+          log.info('Stored "%s" to %s/%s', file.name, options.bucket, id)
+          resolve(id)
+        }
+      })
+    })
+  }
+
+  function getHref(plugin, id, name) {
+    return util.format(
+      '/s/%s/%s%s'
+    , plugin
+    , id
+    , name ? '/' + path.basename(name) : ''
+    )
+  }
+
+  app.post('/s/upload/:plugin', function(req, res, next) {
+    var form = new formidable.IncomingForm()
+    var plugin = req.params.plugin
+    Promise.promisify(form.parse, form)(req)
+      .spread(function(fields, files) {
+        var requests = Object.keys(files).map(function(field) {
+          var file = files[field]
+          log.info('Uploaded "%s" to "%s"', file.name, file.path)
+          return putObject(plugin, file)
+            .then(function (id) {
+              return {
+                field: field
+              , id: id
+              , name: file.name
+              , temppath: file.path
+              }
+            })
+        })
+        return Promise.all(requests)
+      })
+      .then(function(storedFiles) {
+        res.status(201).json({
+          success: true,
+          resources: (function() {
+            var mapped = Object.create(null)
+            storedFiles.forEach(function(file) {
+              var plugin = req.params.plugin
+              mapped[file.field] = {
+                date: new Date()
+              , plugin: plugin
+              , id: file.id
+              , name: file.name
+              , href: getHref(plugin, file.id, file.name)
+              }
+            })
+            return mapped
+          })()
+        })
+        return storedFiles
+      })
+      .then(function (storedFiles){
+        storedFiles.forEach(function (file){
+          fs.unlink(file.temppath)
+        })
+      })
+      .catch(function(err) {
+        log.error('Error storing resource', err.stack)
+        res.status(500)
+          .json({
+            success: false,
+            error: 'ServerError'
+          })
+      })
+  })
+
+  app.get('/s/blob/:id/:name', function(req, res) {
+    var params = {
+      Key: req.params.id,
+      Bucket: options.bucket
+    }
+    s3.getObject(params, function(err, data) {
+      if (err) {
+        log.error('failed to retreive[' + path + ']')
+        log.error(err, err.stack);
+        res.sendStatus(404)
+      } else {
+        res.set({
+          'Content-type': data.ContentType
+        })
+        res.send(data.Body)
+      }
+    })
+  })
+
+  // initialize
+  server.listen(options.port)
+  console.log('Listening on port %d', options.port)
+}

package.json

@@ -33,6 +33,7 @@
     "adbkit": "^2.3.1",
     "adbkit-apkreader": "^1.0.0",
     "adbkit-monkey": "^1.0.1",
+    "aws-sdk": "^2.2.3",
     "bluebird": "^2.9.34",
     "body-parser": "^1.13.3",
     "chalk": "~1.1.1",