Merge branch 'feature/screenshot' into develop

Conflicts:
	res/app/control-panes/dashboard/upload/upload.jade

bower.json

@@ -24,7 +24,7 @@
     "stf-graphics": "git@ghe.amb.ca.local:stf/stf-graphics.git",
     "angular-bootstrap": "~0.11.0",
     "angular-dialog-service": "~5.0.0",
-    "ng-file-upload": "~1.2.11",
+    "ng-file-upload": "~1.4.0",
     "angular-growl-v2": "JanStevens/angular-growl-2#~0.6.0",
     "bluebird": "~1.2.4",
     "angular-tree-control": "~0.1.5",

lib/cli.js

@@ -43,6 +43,9 @@ program
     , 'group timeout'
     , Number
     , 600)
+  .option('-r, --storage-url <url>'
+    , 'URL to storage client'
+    , String)
   .action(function() {
     var serials = cliutil.allUnknownArgs(arguments)
       , options = cliutil.lastArg(arguments)
@@ -53,6 +56,9 @@ program
     if (!options.connectPush) {
       this.missingArgument('--connect-push')
     }
+    if (!options.storageUrl) {
+      this.missingArgument('--storage-url')
+    }
 
     require('./roles/provider')({
       name: options.name
@@ -71,6 +77,7 @@ program
         , '--ports', ports.join(',')
         , '--public-ip', options.publicIp
         , '--group-timeout', options.groupTimeout
+        , '--storage-url', options.storageUrl
         ])
       }
     , endpoints: {
@@ -107,6 +114,9 @@ program
     , 'group timeout'
     , Number
     , 600)
+  .option('-r, --storage-url <url>'
+    , 'URL to storage client'
+    , String)
   .action(function(serial, options) {
     if (!options.connectSub) {
       this.missingArgument('--connect-sub')
@@ -120,6 +130,9 @@ program
     if (!options.ports) {
       this.missingArgument('--ports')
     }
+    if (!options.storageUrl) {
+      this.missingArgument('--storage-url')
+    }
 
     require('./roles/device')({
       serial: serial
@@ -132,6 +145,7 @@ program
       }
     , heartbeatInterval: options.heartbeatInterval
     , groupTimeout: options.groupTimeout * 1000 // change to ms
+    , storageUrl: options.storageUrl
     })
   })
 
@@ -386,6 +400,12 @@ program
   .option('-r, --storage-url <url>'
     , 'URL to storage client'
     , String)
+  .option('--storage-plugin-image-url <url>'
+    , 'URL to image storage plugin'
+    , String)
+  .option('--storage-plugin-apk-url <url>'
+    , 'URL to apk storage plugin'
+    , String)
   .option('-u, --connect-sub <endpoint>'
     , 'sub endpoint'
     , cliutil.list)
@@ -404,6 +424,12 @@ program
     if (!options.storageUrl) {
       this.missingArgument('--storage-url')
     }
+    if (!options.storagePluginImageUrl) {
+      this.missingArgument('--storage-plugin-image-url')
+    }
+    if (!options.storagePluginApkUrl) {
+      this.missingArgument('--storage-plugin-apk-url')
+    }
     if (!options.connectSub) {
       this.missingArgument('--connect-sub')
     }
@@ -417,6 +443,8 @@ program
     , ssid: options.ssid
     , authUrl: options.authUrl
     , storageUrl: options.storageUrl
+    , storagePluginImageUrl: options.storagePluginImageUrl
+    , storagePluginApkUrl: options.storagePluginApkUrl
     , endpoints: {
         sub: options.connectSub
       , push: options.connectPush
@@ -432,34 +460,70 @@ program
     , 'port (or $PORT)'
     , Number
     , process.env.PORT || 7100)
-  .option('--public-ip <ip>'
-    , 'public ip for global access'
-    , String
-    , ip())
   .option('--save-dir <dir>'
     , 'where to save files'
     , String
     , os.tmpdir())
-  .option('--id <id>'
-    , 'communication identifier'
-    , String
-    , 'storage')
-  .option('--connect-push <endpoint>'
-    , 'push endpoint'
-    , cliutil.list)
   .action(function(options) {
-    if (!options.connectPush) {
-      this.missingArgument('--connect-push')
-    }
-
     require('./roles/storage/temp')({
       port: options.port
-    , publicIp: options.publicIp
     , saveDir: options.saveDir
-    , id: options.id
-    , endpoints: {
-        push: options.connectPush
-      }
+    })
+  })
+
+program
+  .command('storage-plugin-image')
+  .description('start storage image plugin')
+  .option('-p, --port <port>'
+    , 'port (or $PORT)'
+    , Number
+    , process.env.PORT || 7100)
+  .option('-r, --storage-url <url>'
+    , 'URL to storage client'
+    , String)
+  .option('-c, --concurrency <num>'
+    , 'maximum number of simultaneous transformations'
+    , Number)
+  .option('--cache-dir <dir>'
+    , 'where to cache images'
+    , String
+    , os.tmpdir())
+  .action(function(options) {
+    if (!options.storageUrl) {
+      this.missingArgument('--storage-url')
+    }
+
+    require('./roles/storage/plugins/image')({
+      port: options.port
+    , storageUrl: options.storageUrl
+    , cacheDir: options.cacheDir
+    , concurrency: options.concurrency || os.cpus().length
+    })
+  })
+
+program
+  .command('storage-plugin-apk')
+  .description('start storage apk plugin')
+  .option('-p, --port <port>'
+    , 'port (or $PORT)'
+    , Number
+    , process.env.PORT || 7100)
+  .option('-r, --storage-url <url>'
+    , 'URL to storage client'
+    , String)
+  .option('--cache-dir <dir>'
+    , 'where to cache images'
+    , String
+    , os.tmpdir())
+  .action(function(options) {
+    if (!options.storageUrl) {
+      this.missingArgument('--storage-url')
+    }
+
+    require('./roles/storage/plugins/apk')({
+      port: options.port
+    , storageUrl: options.storageUrl
+    , cacheDir: options.cacheDir
     })
   })
 
@@ -523,6 +587,14 @@ program
     , 'storage port'
     , Number
     , 7102)
+  .option('--storage-plugin-image-port <port>'
+    , 'storage image plugin port'
+    , Number
+    , 7103)
+  .option('--storage-plugin-apk-port <port>'
+    , 'storage apk plugin port'
+    , Number
+    , 7104)
   .option('--provider <name>'
     , 'provider name (or os.hostname())'
     , String
@@ -593,6 +665,8 @@ program
           , '--connect-push', options.bindDevPull
           , '--group-timeout', options.groupTimeout
           , '--public-ip', options.publicIp
+          , '--storage-url'
+          , util.format('http://localhost:%d/', options.storagePort)
           ].concat(cliutil.allUnknownArgs(args)))
 
         // auth-mock
@@ -611,6 +685,10 @@ program
           , '--auth-url', util.format('http://localhost:%d/', options.authPort)
           , '--storage-url'
           , util.format('http://localhost:%d/', options.storagePort)
+          , '--storage-plugin-image-url'
+          , util.format('http://localhost:%d/', options.storagePluginImagePort)
+          , '--storage-plugin-apk-url'
+          , util.format('http://localhost:%d/', options.storagePluginApkPort)
           , '--connect-sub', options.bindAppPub
           , '--connect-push', options.bindAppPull
           ].concat((function() {
@@ -625,7 +703,22 @@ program
       , procutil.fork(__filename, [
             'storage-temp'
           , '--port', options.storagePort
-          , '--connect-push', options.bindDevPull
+          ])
+
+        // image processor
+      , procutil.fork(__filename, [
+            'storage-plugin-image'
+          , '--port', options.storagePluginImagePort
+          , '--storage-url'
+          , util.format('http://localhost:%d/', options.storagePort)
+          ])
+
+        // apk processor
+      , procutil.fork(__filename, [
+            'storage-plugin-apk'
+          , '--port', options.storagePluginApkPort
+          , '--storage-url'
+          , util.format('http://localhost:%d/', options.storagePort)
           ])
       ]
 

lib/roles/app.js

@@ -73,14 +73,22 @@ module.exports = function(options) {
   , authUrl: options.authUrl
   }))
 
-  // Proxied requests must come before any body parsers
-  app.post('/api/v1/resources', function(req, res) {
+  // Proxied requests must come before any body parsers. These proxies are
+  // here mainly for convenience, they should be replaced with proper reverse
+  // proxies in production.
+  app.all('/api/v1/s/image/*', function(req, res) {
     proxy.web(req, res, {
-      target: options.storageUrl
+      target: options.storagePluginImageUrl
     })
   })
 
-  app.get('/api/v1/resources/:id', function(req, res) {
+  app.all('/api/v1/s/apk/*', function(req, res) {
+    proxy.web(req, res, {
+      target: options.storagePluginApkUrl
+    })
+  })
+
+  app.all('/api/v1/s/*', function(req, res) {
     proxy.web(req, res, {
       target: options.storageUrl
     })
@@ -507,7 +515,11 @@ module.exports = function(options) {
             channel
           , wireutil.transaction(
               responseChannel
-            , new wire.InstallMessage(data)
+            , new wire.InstallMessage(
+                data.href
+              , data.launch === true
+              , JSON.stringify(data.manifest)
+              )
             )
           ])
         })
@@ -637,6 +649,16 @@ module.exports = function(options) {
             )
           ])
         })
+        .on('screen.capture', function(channel, responseChannel) {
+          joinChannel(responseChannel)
+          push.send([
+            channel
+          , wireutil.transaction(
+              responseChannel
+            , new wire.ScreenCaptureMessage()
+            )
+          ])
+        })
     })
     .finally(function() {
       // Clean up all listeners and subscriptions

lib/roles/device.js

@@ -19,6 +19,7 @@ module.exports = function(options) {
         .dependency(require('./device/plugins/solo'))
         .dependency(require('./device/plugins/heartbeat'))
         .dependency(require('./device/plugins/display'))
+        .dependency(require('./device/plugins/screenshot'))
         .dependency(require('./device/plugins/http'))
         .dependency(require('./device/plugins/service'))
         .dependency(require('./device/plugins/browser'))

lib/roles/device/plugins/install.js

@@ -1,4 +1,6 @@
 var stream = require('stream')
+var url = require('url')
+var util = require('util')
 
 var syrup = require('syrup')
 var request = require('request')
@@ -17,7 +19,7 @@ module.exports = syrup.serial()
     var log = logger.createLogger('device:plugins:install')
 
     router.on(wire.InstallMessage, function(channel, message) {
-      log.info('Installing "%s"', message.url)
+      log.info('Installing "%s"', message.href)
 
       var reply = wireutil.reply(options.serial)
 
@@ -30,7 +32,7 @@ module.exports = syrup.serial()
 
       function pushApp() {
         var req = request({
-          url: message.url
+          url: url.resolve(options.storageUrl, message.href)
         })
 
         // We need to catch the Content-Length on the fly or we risk
@@ -104,16 +106,30 @@ module.exports = syrup.serial()
             .timeout(30000)
         })
         .then(function() {
-          if (message.launchActivity) {
-            log.info(
-              'Launching activity with action "%s" on component "%s"'
-            , message.launchActivity.action
-            , message.launchActivity.component
-            )
-            // Progress 90%
-            sendProgress('launching_app', 90)
-            return adb.startActivity(options.serial, message.launchActivity)
-              .timeout(15000)
+          if (message.launch) {
+            var manifest = JSON.parse(message.manifest)
+            if (manifest.application.launcherActivities.length) {
+              var launchActivity = {
+                action: 'android.intent.action.MAIN'
+              , component: util.format(
+                  '%s/%s'
+                , manifest.package
+                , manifest.application.launcherActivities[0].name
+                )
+              , category: ['android.intent.category.LAUNCHER']
+              , flags: 0x10200000
+              }
+
+              log.info(
+                'Launching activity with action "%s" on component "%s"'
+              , launchActivity.action
+              , launchActivity.component
+              )
+              // Progress 90%
+              sendProgress('launching_app', 90)
+              return adb.startActivity(options.serial, launchActivity)
+                .timeout(15000)
+            }
           }
         })
         .then(function() {

lib/roles/device/plugins/screenshot.js

@@ -0,0 +1,66 @@
+var http = require('http')
+var util = require('util')
+
+var syrup = require('syrup')
+var Promise = require('bluebird')
+
+var logger = require('../../../util/logger')
+var wire = require('../../../wire')
+var wireutil = require('../../../wire/util')
+
+module.exports = syrup.serial()
+  .dependency(require('../support/router'))
+  .dependency(require('../support/push'))
+  .dependency(require('../support/storage'))
+  .dependency(require('./display'))
+  .define(function(options, router, push, storage, display) {
+    var log = logger.createLogger('device:plugins:screenshot')
+    var plugin = Object.create(null)
+
+    plugin.capture = function() {
+      log.info('Capturing screenshot from %s', display.url)
+
+      return new Promise(function(resolve, reject) {
+        var req = http.get(display.url)
+
+        function responseListener(res) {
+          if (res.statusCode !== 200) {
+            reject(new Error(util.format(
+              'Screenshot capture failed: HTTP %d'
+            , res.statusCode
+            )))
+          }
+          else {
+            resolve(storage.store('image', res, {
+              filename: util.format('%s.jpg', options.serial)
+            , contentType: 'image/jpeg'
+            , knownLength: +res.headers['content-length']
+            }))
+          }
+        }
+
+        req.on('response', responseListener)
+        req.on('error', reject)
+      })
+    }
+
+    router.on(wire.ScreenCaptureMessage, function(channel) {
+      var reply = wireutil.reply(options.serial)
+      plugin.capture()
+        .then(function(file) {
+          push.send([
+            channel
+          , reply.okay('success', file)
+          ])
+        })
+        .catch(function(err) {
+          log.error('Screen capture failed', err.stack)
+          push.send([
+            channel
+          , reply.fail(err.message)
+          ])
+        })
+    })
+
+    return plugin
+  })

lib/roles/device/support/storage.js

@@ -0,0 +1,53 @@
+var util = require('util')
+
+var syrup = require('syrup')
+var Promise = require('bluebird')
+var request = require('request')
+
+var logger = require('../../../util/logger')
+
+module.exports = syrup.serial()
+  .define(function(options) {
+    var log = logger.createLogger('device:support:storage')
+    var plugin = Object.create(null)
+
+    plugin.store = function(type, stream, meta) {
+      var resolver = Promise.defer()
+
+      var req = request.post({
+          url: util.format('%sapi/v1/s/%s', options.storageUrl, type)
+        }
+      , function(err, res, body) {
+          if (err) {
+            log.error('Upload failed', err.stack)
+            resolver.reject(err)
+          }
+          else if (res.statusCode !== 201) {
+            log.error('Upload failed: HTTP %d', res.statusCode)
+            resolver.reject(new Error(util.format(
+              'Upload failed: HTTP %d'
+            , res.statusCode
+            )))
+          }
+          else {
+            try {
+              var result = JSON.parse(body)
+              log.info('Uploaded to %s', result.resources.file.href)
+              resolver.resolve(result.resources.file)
+            }
+            catch (err) {
+              log.error('Invalid JSON in response', err.stack, body)
+              resolver.reject(err)
+            }
+          }
+        }
+      )
+
+      req.form()
+        .append('file', stream, meta)
+
+      return resolver.promise
+    }
+
+    return plugin
+  })

lib/roles/storage/plugins/apk/index.js

@@ -0,0 +1,54 @@
+var http = require('http')
+var url = require('url')
+
+var express = require('express')
+var httpProxy = require('http-proxy')
+
+var logger = require('../../../../util/logger')
+var download = require('../../../../util/download')
+var manifest = require('./task/manifest')
+
+module.exports = function(options) {
+  var log = logger.createLogger('storage:plugins:apk')
+    , app = express()
+    , server = http.createServer(app)
+    , proxy = httpProxy.createProxyServer()
+
+  proxy.on('error', function(err) {
+    log.error('Proxy had an error', err.stack)
+  })
+
+  app.set('strict routing', true)
+  app.set('case sensitive routing', true)
+  app.set('trust proxy', true)
+
+  app.get('/api/v1/s/apk/:id/*/manifest', function(req, res) {
+    download(url.resolve(options.storageUrl, req.url), {
+        dir: options.cacheDir
+      })
+      .then(manifest)
+      .then(function(data) {
+        res.status(200)
+          .json({
+            success: true
+          , manifest: data
+          })
+      })
+      .catch(function(err) {
+        log.error('Unable to read manifest of "%s"', req.params.id, err.stack)
+        res.status(500)
+          .json({
+            success: false
+          })
+      })
+  })
+
+  app.get('/api/v1/s/apk/:id/*', function(req, res) {
+    proxy.web(req, res, {
+      target: options.storageUrl
+    })
+  })
+
+  server.listen(options.port)
+  log.info('Listening on port %d', options.port)
+}

lib/roles/storage/plugins/apk/task/manifest.js

@@ -0,0 +1,19 @@
+var Promise = require('bluebird')
+var ApkReader = require('adbkit-apkreader')
+
+module.exports = function(file) {
+  var resolver = Promise.defer()
+
+  process.nextTick(function() {
+    try {
+      var reader = ApkReader.readFile(file.path)
+      var manifest = reader.readManifestSync()
+      resolver.resolve(manifest)
+    }
+    catch (err) {
+      resolver.reject(err)
+    }
+  })
+
+  return resolver.promise
+}

lib/roles/storage/plugins/image/index.js

@@ -0,0 +1,52 @@
+var http = require('http')
+
+var express = require('express')
+
+var logger = require('../../../../util/logger')
+var requtil = require('../../../../util/requtil')
+
+var parseCrop = require('./param/crop')
+var parseGravity = require('./param/gravity')
+var get = require('./task/get')
+var transform = require('./task/transform')
+
+module.exports = function(options) {
+  var log = logger.createLogger('storage:plugins:image')
+    , app = express()
+    , server = http.createServer(app)
+
+  app.set('strict routing', true)
+  app.set('case sensitive routing', true)
+  app.set('trust proxy', true)
+
+  app.get(
+    '/api/v1/s/image/:id/*'
+  , requtil.limit(options.concurrency, function(req, res) {
+      return get(req.url, options)
+        .then(function(stream) {
+          return transform(stream, {
+            crop: parseCrop(req.query.crop)
+          , gravity: parseGravity(req.query.gravity)
+          })
+        })
+        .then(function(out) {
+          res.status(200)
+          out.pipe(res)
+        })
+        .catch(function(err) {
+          log.error(
+            'Unable to transform resource "%s"'
+          , req.params.id
+          , err.stack
+          )
+          res.status(500)
+            .json({
+              success: false
+            })
+        })
+    })
+  )
+
+  server.listen(options.port)
+  log.info('Listening on port %d', options.port)
+}

lib/roles/storage/plugins/image/param/crop.js

@@ -0,0 +1,14 @@
+var RE_CROP = /^([0-9]*)x([0-9]*)$/
+
+module.exports = function(raw) {
+  var parsed
+
+  if (raw && (parsed = RE_CROP.exec(raw))) {
+    return {
+      width: +parsed[1] || 0
+    , height: +parsed[2] || 0
+    }
+  }
+
+  return null
+}

lib/roles/storage/plugins/image/param/gravity.js

@@ -0,0 +1,21 @@
+var GRAVITY = {
+  northwest: 'NorthWest'
+, north: 'North'
+, northeast: 'NorthEast'
+, west: 'West'
+, center: 'Center'
+, east: 'East'
+, southwest: 'SouthWest'
+, south: 'South'
+, southeast: 'SouthEast'
+}
+
+module.exports = function(raw) {
+  var parsed
+
+  if (raw && (parsed = GRAVITY[raw])) {
+    return parsed
+  }
+
+  return null
+}

lib/roles/storage/plugins/image/task/get.js

@@ -0,0 +1,20 @@
+var util = require('util')
+var http = require('http')
+var url = require('url')
+
+var Promise = require('bluebird')
+
+module.exports = function(path, options) {
+  return new Promise(function(resolve, reject) {
+    http.get(url.resolve(options.storageUrl, path))
+      .on('response', function(res) {
+        if (res.statusCode !== 200) {
+          reject(new Error(util.format('HTTP %d', res.statusCode)))
+        }
+        else {
+          resolve(res)
+        }
+      })
+      .on('error', reject)
+  })
+}

lib/roles/storage/plugins/image/task/transform.js

@@ -0,0 +1,26 @@
+var gm = require('gm')
+var Promise = require('bluebird')
+
+module.exports = function(stream, options) {
+  return new Promise(function(resolve, reject) {
+    var transform = gm(stream)
+
+    if (options.gravity) {
+      transform.gravity(options.gravity)
+    }
+
+    if (options.crop) {
+      transform.geometry(options.crop.width, options.crop.height, '^')
+      transform.crop(options.crop.width, options.crop.height, 0, 0)
+    }
+
+    transform.stream(function(err, stdout) {
+      if (err) {
+        reject(err)
+      }
+      else {
+        resolve(stdout)
+      }
+    })
+  })
+}

lib/roles/storage/temp.js

@@ -1,35 +1,23 @@
 var http = require('http')
 var util = require('util')
-var fs = require('fs')
+var path = require('path')
 
 var express = require('express')
 var validator = require('express-validator')
 var formidable = require('formidable')
 var Promise = require('bluebird')
-var ApkReader = require('adbkit-apkreader')
-var request = require('request')
-var progress = require('request-progress')
-var temp = require('temp')
-var zmq = require('zmq')
 
 var logger = require('../../util/logger')
-var requtil = require('../../util/requtil')
 var Storage = require('../../util/storage')
-var wireutil = require('../../wire/util')
+var requtil = require('../../util/requtil')
+var download = require('../../util/download')
 
 module.exports = function(options) {
-  var log = logger.createLogger('storage-temp')
+  var log = logger.createLogger('storage:temp')
     , app = express()
     , server = http.createServer(app)
     , storage = new Storage()
 
-  // Output
-  var push = zmq.socket('push')
-  options.endpoints.push.forEach(function(endpoint) {
-    log.info('Sending output to %s', endpoint)
-    push.connect(endpoint)
-  })
-
   app.set('strict routing', true)
   app.set('case sensitive routing', true)
   app.set('trust proxy', true)
@@ -41,175 +29,42 @@ module.exports = function(options) {
     log.info('Cleaning up inactive resource "%s"', id)
   })
 
-  function processFile(file) {
-    var resolver = Promise.defer()
-
-    log.info('Processing file "%s"', file.path)
-
-    resolver.progress({
-      percent: 0
-    })
-
-    process.nextTick(function() {
-      try {
-        var reader = ApkReader.readFile(file.path)
-        var manifest = reader.readManifestSync()
-        resolver.resolve(manifest)
-      }
-      catch (err) {
-        err.reportCode = 'fail_invalid_app_file'
-        resolver.reject(err)
-      }
-    })
-
-    return resolver.promise
-  }
-
-  function storeFile(file) {
-    var id = storage.store(file)
-    return Promise.resolve({
-      id: id
-    , url: util.format(
-        'http://%s:%s/api/v1/resources/%s'
-      , options.publicIp
-      , options.port
-      , id
-      )
-    })
-  }
-
-  function download(url) {
-    var resolver = Promise.defer()
-    var path = temp.path({
-      dir: options.saveDir
-    })
-
-    log.info('Downloading "%s" to "%s"', url, path)
-
-    function errorListener(err) {
-      err.reportCode = 'fail_download'
-      resolver.reject(err)
-    }
-
-    function progressListener(state) {
-      resolver.progress(state)
-    }
-
-    function closeListener() {
-      resolver.resolve({
-        path: path
-      })
-    }
-
-    resolver.progress({
-      percent: 0
-    })
-
-    try {
-      var req = progress(request(url), {
-          throttle: 100 // Throttle events, not upload speed
-        })
-        .on('progress', progressListener)
-
-      var save = req.pipe(fs.createWriteStream(path))
-        .on('error', errorListener)
-        .on('close', closeListener)
-    }
-    catch (err) {
-      err.reportCode = 'fail_invalid_url'
-      resolver.reject(err)
-    }
-
-    return resolver.promise.finally(function() {
-      req.removeListener('progress', progressListener)
-      save.removeListener('error', errorListener)
-      save.removeListener('close', closeListener)
-    })
-  }
-
-  app.post('/api/v1/resources', function(req, res) {
-    var reply = wireutil.reply(options.id)
-
-    function sendProgress(data, progress) {
-      if (req.query.channel) {
-        push.send([
-          req.query.channel
-        , reply.progress(data, progress)
-        ])
-      }
-    }
-
-    function sendDone(success, data, body) {
-      if (req.query.channel) {
-        push.send([
-          req.query.channel
-        , reply.okay(data, body)
-        ])
-      }
-    }
-
+  app.post('/api/v1/s/:type/download', function(req, res) {
     requtil.validate(req, function() {
-        req.checkQuery('channel').notEmpty()
+        req.checkBody('url').notEmpty()
       })
       .then(function() {
-        if (req.is('application/json')) {
-          return requtil.validate(req, function() {
-              req.checkBody('url').notEmpty()
-            })
-            .then(function() {
-              return download(req.body.url)
-                .progressed(function(progress) {
-                  sendProgress('uploading', 0.7 * progress.percent)
-                })
-            })
-        }
-        else {
-          var form = Promise.promisifyAll(new formidable.IncomingForm())
-          var progressListener = function(received, expected) {
-            if (expected) {
-              sendProgress('uploading', 70 * (received / expected))
-            }
-          }
-          sendProgress('uploading', 0)
-          form.on('progress', progressListener)
-          return form.parseAsync(req)
-            .finally(function() {
-              form.removeListener('progress', progressListener)
-            })
-            .spread(function(fields, files) {
-              if (!files.file) {
-                throw new requtil.ValidationError('validation error', [
-                  {
-                    "param": "file"
-                  , "msg": "Required value"
-                  }
-                ])
-              }
-              return files.file
-            })
+        return download(req.body.url, {
+          dir: options.cacheDir
+        })
+      })
+      .then(function(file) {
+        return {
+          id: storage.store(file)
+        , name: file.name
         }
       })
       .then(function(file) {
-        return processFile(file)
-          .progressed(function(progress) {
-            sendProgress('processing', 70 + 0.2 * progress.percent)
+        res.status(201)
+          .json({
+            success: true
+          , resource: {
+              date: new Date()
+            , type: req.params.type
+            , id: file.id
+            , name: file.name
+            , href: util.format(
+                '/api/v1/s/%s/%s%s'
+              , req.params.type
+              , file.id
+              , file.name
+                  ? util.format('/%s', path.basename(file.name))
+                  : ''
+              )
+            }
           })
-          .then(function(manifest) {
-            sendProgress('storing', 90)
-            return storeFile(file)
-              .then(function(data) {
-                data.manifest = manifest
-                return data
-              })
-          })
-      })
-      .then(function(data) {
-        sendDone(true, 'success', data)
-        data.success = true
-        res.json(201, data)
       })
       .catch(requtil.ValidationError, function(err) {
-        sendDone(false, err.reportCode || 'fail_validation')
         res.status(400)
           .json({
             success: false
@@ -218,8 +73,7 @@ module.exports = function(options) {
           })
       })
       .catch(function(err) {
-        log.error('Unexpected error', err.stack)
-        sendDone(false, err.reportCode || 'fail')
+        log.error('Error storing resource', err.stack)
         res.status(500)
           .json({
             success: false
@@ -228,7 +82,57 @@ module.exports = function(options) {
       })
   })
 
-  app.get('/api/v1/resources/:id', function(req, res) {
+  app.post('/api/v1/s/:type', function(req, res) {
+    var form = new formidable.IncomingForm()
+    Promise.promisify(form.parse, form)(req)
+      .spread(function(fields, files) {
+        return Object.keys(files).map(function(field) {
+          var file = files[field]
+          log.info('Uploaded "%s" to "%s"', file.name, file.path)
+          return {
+            field: field
+          , id: storage.store(file)
+          , name: file.name
+          }
+        })
+      })
+      .then(function(storedFiles) {
+        res.status(201)
+          .json({
+            success: true
+          , resources: (function() {
+              var mapped = Object.create(null)
+              storedFiles.forEach(function(file) {
+                mapped[file.field] = {
+                  date: new Date()
+                , type: req.params.type
+                , id: file.id
+                , name: file.name
+                , href: util.format(
+                    '/api/v1/s/%s/%s%s'
+                  , req.params.type
+                  , file.id
+                  , file.name
+                      ? util.format('/%s', path.basename(file.name))
+                      : ''
+                  )
+                }
+              })
+              return mapped
+            })()
+          })
+      })
+      .catch(function(err) {
+        log.error('Error storing resource', err.stack)
+        res.status(500)
+          .json({
+            success: false
+          , error: 'ServerError'
+          })
+      })
+  })
+
+  app.get('/api/v1/s/:type/:id/*', function(req, res) {
     var file = storage.retrieve(req.params.id)
     if (file) {
       res.set('Content-Type', file.type)

lib/util/download.js

@@ -0,0 +1,67 @@
+var fs = require('fs')
+
+var Promise = require('bluebird')
+var request = require('request')
+var progress = require('request-progress')
+var temp = require('temp')
+
+module.exports = function download(url, options) {
+  var resolver = Promise.defer()
+  var path = temp.path(options)
+
+  function errorListener(err) {
+    resolver.reject(err)
+  }
+
+  function progressListener(state) {
+    if (state.total !== null) {
+      resolver.progress({
+        lengthComputable: true
+      , loaded: state.received
+      , total: state.total
+      })
+    }
+    else {
+      resolver.progress({
+        lengthComputable: false
+      , loaded: state.received
+      , total: state.received
+      })
+    }
+  }
+
+  function closeListener() {
+    resolver.resolve({
+      path: path
+    })
+  }
+
+  resolver.progress({
+    percent: 0
+  })
+
+  try {
+    var req = progress(request(url), {
+        throttle: 100 // Throttle events, not upload speed
+      })
+      .on('progress', progressListener)
+
+    resolver.promise.finally(function() {
+      req.removeListener('progress', progressListener)
+    })
+
+    var save = req.pipe(fs.createWriteStream(path))
+      .on('error', errorListener)
+      .on('close', closeListener)
+
+    resolver.promise.finally(function() {
+      save.removeListener('error', errorListener)
+      save.removeListener('close', closeListener)
+    })
+  }
+  catch (err) {
+    resolver.reject(err)
+  }
+
+  return resolver.promise
+}

lib/util/requtil.js

@@ -26,3 +26,25 @@ module.exports.validate = function(req, rules) {
     }
   })
 }
+
+module.exports.limit = function(limit, handler) {
+  var queue = []
+  var running = 0
+
+  function done() {
+    running -= 1
+    maybeNext()
+  }
+
+  function maybeNext() {
+    while (running < limit && queue.length) {
+      running += 1
+      handler.apply(null, queue.shift()).finally(done)
+    }
+  }
+
+  return function() {
+    queue.push(arguments)
+    maybeNext()
+  }
+}

lib/util/storage.js

@@ -14,14 +14,18 @@ util.inherits(Storage, events.EventEmitter)
 
 Storage.prototype.store = function(file) {
   var id = uuid.v4()
+  this.set(id, file)
+  return id
+}
 
+Storage.prototype.set = function(id, file) {
   this.files[id] = {
     timeout: 600000
   , lastActivity: Date.now()
   , data: file
   }
 
-  return id
+  return file
 }
 
 Storage.prototype.remove = function(id) {

lib/wire/wire.proto

@@ -50,6 +50,7 @@ enum MessageType {
   PhoneStateEvent            = 47;
   RotationEvent              = 48;
   StoreOpenMessage           = 49;
+  ScreenCaptureMessage       = 50;
 }
 
 message Envelope {
@@ -353,8 +354,9 @@ message ShellKeepAliveMessage {
 }
 
 message InstallMessage {
-  required string url = 1;
-  optional LaunchActivityMessage launchActivity = 2;
+  required string href = 1;
+  required bool launch = 2;
+  optional string manifest = 3;
 }
 
 message UninstallMessage {
@@ -400,6 +402,9 @@ message BrowserClearMessage {
 message StoreOpenMessage {
 }
 
+message ScreenCaptureMessage {
+}
+
 // Events, these must be kept in sync with STFService/wire.proto
 
 message AirplaneModeEvent {

res/app/app.js

@@ -3,7 +3,8 @@ require('angular-route')
 require('angular-touch')
 
 require('angular-gettext')
-require('ng-file-upload')
+require('ng-file-upload-shim5')
+require('ng-file-upload-main')
 
 angular.module('app', [
   'ngRoute',

res/app/components/stf/control/control-service.js

@@ -111,49 +111,8 @@ module.exports = function ControlServiceFactory(
       return sendTwoWay('device.identify')
     }
 
-    this.uploadUrl = function(url) {
-      var tx = TransactionService.create({
-        id: 'storage'
-      })
-      socket.emit('storage.upload', channel, tx.channel, {
-        url: url
-      })
-      return tx.promise
-    }
-
-    this.uploadFile = function(files) {
-      if (files.length !== 1) {
-        throw new Error('Can only upload one file')
-      }
-      var tx = TransactionService.create({
-        id: 'storage'
-      })
-      TransactionService.punch(tx.channel)
-        .then(function() {
-          $upload.upload({
-            url: '/api/v1/resources?channel=' + tx.channel
-          , method: 'POST'
-          , file: files[0]
-          })
-        })
-      return tx.promise
-    }
-
     this.install = function(options) {
-      var app = options.manifest.application
-      var params = {
-        url: options.url
-      }
-      if (app.launcherActivities.length) {
-        var activity = app.launcherActivities[0]
-        params.launchActivity = {
-          action: 'android.intent.action.MAIN'
-        , component: options.manifest.package + '/' + activity.name
-        , category: ['android.intent.category.LAUNCHER']
-        , flags: 0x10200000
-        }
-      }
-      return sendTwoWay('device.install', params)
+      return sendTwoWay('device.install', options)
     }
 
     this.uninstall = function(pkg) {
@@ -216,6 +175,10 @@ module.exports = function ControlServiceFactory(
       return sendTwoWay('store.open')
     }
 
+    this.screenshot = function() {
+      return sendTwoWay('screen.capture')
+    }
+
     window.cc = this
   }
 

res/app/components/stf/control/index.js

@@ -3,3 +3,4 @@ module.exports = angular.module('stf/control', [
 ])
   .factory('TransactionService', require('./transaction-service'))
   .factory('ControlService', require('./control-service'))
+  .factory('StorageService', require('./storage-service'))

res/app/components/stf/control/storage-service.js

@@ -0,0 +1,40 @@
+var Promise = require('bluebird')
+
+module.exports = function StorageServiceFactory($http, $upload) {
+  var service = {}
+
+  service.storeUrl = function(type, url) {
+    return $http({
+      url: '/api/v1/s/' + type + '/download'
+    , method: 'POST'
+    , data: {
+        url: url
+      }
+    })
+  }
+
+  service.storeFile = function(type, files) {
+    var resolver = Promise.defer()
+
+    $upload.upload({
+        url: '/api/v1/s/' + type
+      , method: 'POST'
+      , file: files
+      })
+      .then(
+        function(value) {
+          resolver.resolve(value)
+        }
+      , function(err) {
+          resolver.reject(err)
+        }
+      , function(progressEvent) {
+          resolver.progress(progressEvent)
+        }
+      )
+
+    return resolver.promise
+  }
+
+  return service
+}

res/app/control-panes/dashboard/upload/upload-controller.js

@@ -1,5 +1,9 @@
-module.exports = function UploadCtrl($scope, SettingsService, gettext) {
-
+module.exports = function UploadCtrl(
+  $scope
+, $http
+, SettingsService
+, StorageService
+) {
   $scope.upload = null
   $scope.installation = null
   $scope.installEnabled = true
@@ -35,23 +39,49 @@ module.exports = function UploadCtrl($scope, SettingsService, gettext) {
 
   $scope.installFile = function ($files) {
     $scope.upload = {
-      progress: 0,
-      lastData: 'uploading'
+      progress: 0
+    , lastData: 'uploading'
     }
 
-    $scope.installation = null
-    return $scope.control.uploadFile($files)
-      .progressed(function (uploadResult) {
-        $scope.$apply(function () {
-          $scope.upload = uploadResult
-        })
+    return StorageService.storeFile('apk', $files)
+      .progressed(function(e) {
+        if (e.lengthComputable) {
+          $scope.upload = {
+            progress: e.loaded / e.total * 100
+          , lastData: 'uploading'
+          }
+        }
       })
-      .then(function (uploadResult) {
-        $scope.$apply(function () {
-          $scope.upload = uploadResult
-        })
-        if (uploadResult.success) {
-          return $scope.maybeInstall(uploadResult.body)
+      .then(function(res) {
+        $scope.upload = {
+          progress: 100
+        , lastData: 'processing'
+        }
+
+        var href = res.data.resources.file0.href
+        return $http.get(href + '/manifest')
+          .then(function(res) {
+            $scope.upload = {
+              progress: 100
+            , lastData: 'success'
+            , settled: true
+            }
+
+            if (res.data.success) {
+              return $scope.maybeInstall({
+                href: href
+              , launch: $scope.launchEnabled
+              , manifest: res.data.manifest
+              })
+            }
+          })
+      })
+      .catch(function(err) {
+        console.log('Upload error', err)
+        $scope.upload = {
+          progress: 100
+        , lastData: 'fail'
+        , settled: true
         }
       })
   }

res/app/control-panes/dashboard/upload/upload.jade

@@ -3,9 +3,9 @@
     i.fa.fa-upload
     span(translate) Upload
     clear-button(ng-click='clear()', ng-disabled='!installation && !upload').btn-xs
-    //label.checkbox-inline.pull-right
-      input(type='checkbox', ng-model='launchEnabled', ng-disabled='true')
-      span(translate) Launch
+    label.checkbox-inline.pull-right
+      input(type='checkbox', ng-model='launchEnabled')
+      span Launch
     label.checkbox-inline.pull-right
       input(type='checkbox', ng-model='installEnabled')
       span(translate) Install
@@ -65,8 +65,6 @@
                 span(translate) Uploading...
               strong(ng-switch-when='processing')
                 span(translate) Processing...
-              strong(ng-switch-when='storing')
-                span(translate) Storing...
               strong(ng-switch-when='fail')
                 span(translate) Upload failed
               strong(ng-switch-when='success')

res/app/control-panes/screenshots/screenshots-controller.js

@@ -1,3 +1,31 @@
-module.exports = function ScreenshotsCtrl($scope) {
-    
+module.exports = function ScreenshotsCtrl($scope, SettingsService) {
+  $scope.screenshots = []
+  $scope.shotSize = 'small'
+
+  $scope.clear = function () {
+    $scope.screenshots = []
+  }
+
+  SettingsService.bind($scope, {
+    key: 'shotSize'
+  , storeName: 'ScreenShots.shotSize'
+  })
+
+  $scope.shotSizeUrlParameter = function () {
+    var sizes = {
+      'small': '?crop=100x',
+      'medium': '?crop=320x',
+      'large': '?crop=450x',
+      'original': ''
+    }
+    return sizes[$scope.shotSize]
+  }
+
+  $scope.takeScreenShot = function () {
+    $scope.control.screenshot().then(function(result) {
+      $scope.$apply(function() {
+        $scope.screenshots.push(result)
+      })
+    })
+  }
 }

res/app/control-panes/screenshots/screenshots.jade

@@ -24,9 +24,8 @@
     nothing-to-show(message='{{"No screenshots taken"|translate}}', icon='fa-camera', ng-show='!screenshots.length')
     ul.screenshots-icon-view.clear-fix
       li(ng-repeat='shot in screenshots').cursor-select
-        h4 {{ shot.device.capabilities.info.name.id || shot.device.capabilities.model }}
-        h5 {{ shot.value.date | date:'yyyy/MM/dd HH:mm:ss'  }}
-        a(ng-href='{{ shot.value.screenshotUrl }}', target='_blank')
-          img(ng-src='{{ shot.value.screenshotUrl + shotSizeUrlParameter() }}')
+        h5 {{ shot.body.date | date:'yyyy/MM/dd HH:mm:ss'  }}
+        a(ng-href='{{ shot.body.href }}', target='_blank')
+          img(ng-src='{{ shot.body.href + shotSizeUrlParameter() }}')
 
-    .clearfix
+    .clearfix

webpack.config.js

@@ -26,7 +26,8 @@ module.exports = {
     , 'localforage': 'localforage/dist/localforage.js'
     , 'socket.io': 'socket.io-client/dist/socket.io'
     , 'oboe': 'oboe/dist/oboe-browser'
-    , 'ng-file-upload': 'ng-file-upload/angular-file-upload'
+    , 'ng-file-upload-shim5': 'ng-file-upload/angular-file-upload-html5-shim'
+    , 'ng-file-upload-main': 'ng-file-upload/angular-file-upload'
     , 'bluebird': 'bluebird/js/browser/bluebird'
     }
   }