diff --git a/bower.json b/bower.json index 757b4f1a..326cda61 100644 --- a/bower.json +++ b/bower.json @@ -24,7 +24,7 @@ "stf-graphics": "git@ghe.amb.ca.local:stf/stf-graphics.git", "angular-bootstrap": "~0.11.0", "angular-dialog-service": "~5.0.0", - "ng-file-upload": "~1.2.11", + "ng-file-upload": "~1.4.0", "angular-growl-v2": "JanStevens/angular-growl-2#~0.6.0", "bluebird": "~1.2.4", "angular-tree-control": "~0.1.5", diff --git a/lib/cli.js b/lib/cli.js index 569d71d4..169e147f 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -43,6 +43,9 @@ program , 'group timeout' , Number , 600) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) .action(function() { var serials = cliutil.allUnknownArgs(arguments) , options = cliutil.lastArg(arguments) @@ -53,6 +56,9 @@ program if (!options.connectPush) { this.missingArgument('--connect-push') } + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } require('./roles/provider')({ name: options.name @@ -71,6 +77,7 @@ program , '--ports', ports.join(',') , '--public-ip', options.publicIp , '--group-timeout', options.groupTimeout + , '--storage-url', options.storageUrl ]) } , endpoints: { @@ -107,6 +114,9 @@ program , 'group timeout' , Number , 600) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) .action(function(serial, options) { if (!options.connectSub) { this.missingArgument('--connect-sub') @@ -120,6 +130,9 @@ program if (!options.ports) { this.missingArgument('--ports') } + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } require('./roles/device')({ serial: serial @@ -132,6 +145,7 @@ program } , heartbeatInterval: options.heartbeatInterval , groupTimeout: options.groupTimeout * 1000 // change to ms + , storageUrl: options.storageUrl }) }) @@ -386,6 +400,12 @@ program .option('-r, --storage-url ' , 'URL to storage client' , String) + .option('--storage-plugin-image-url ' + , 'URL to image storage plugin' + , String) + .option('--storage-plugin-apk-url ' + , 'URL to apk storage plugin' + , String) .option('-u, --connect-sub ' , 'sub endpoint' , cliutil.list) @@ -404,6 +424,12 @@ program if (!options.storageUrl) { this.missingArgument('--storage-url') } + if (!options.storagePluginImageUrl) { + this.missingArgument('--storage-plugin-image-url') + } + if (!options.storagePluginApkUrl) { + this.missingArgument('--storage-plugin-apk-url') + } if (!options.connectSub) { this.missingArgument('--connect-sub') } @@ -417,6 +443,8 @@ program , ssid: options.ssid , authUrl: options.authUrl , storageUrl: options.storageUrl + , storagePluginImageUrl: options.storagePluginImageUrl + , storagePluginApkUrl: options.storagePluginApkUrl , endpoints: { sub: options.connectSub , push: options.connectPush @@ -432,34 +460,70 @@ program , 'port (or $PORT)' , Number , process.env.PORT || 7100) - .option('--public-ip ' - , 'public ip for global access' - , String - , ip()) .option('--save-dir ' , 'where to save files' , String , os.tmpdir()) - .option('--id ' - , 'communication identifier' - , String - , 'storage') - .option('--connect-push ' - , 'push endpoint' - , cliutil.list) .action(function(options) { - if (!options.connectPush) { - this.missingArgument('--connect-push') - } - require('./roles/storage/temp')({ port: options.port - , publicIp: options.publicIp , saveDir: options.saveDir - , id: options.id - , endpoints: { - push: options.connectPush - } + }) + }) + +program + .command('storage-plugin-image') + .description('start storage image plugin') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) + .option('-c, --concurrency ' + , 'maximum number of simultaneous transformations' + , Number) + .option('--cache-dir ' + , 'where to cache images' + , String + , os.tmpdir()) + .action(function(options) { + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } + + require('./roles/storage/plugins/image')({ + port: options.port + , storageUrl: options.storageUrl + , cacheDir: options.cacheDir + , concurrency: options.concurrency || os.cpus().length + }) + }) + +program + .command('storage-plugin-apk') + .description('start storage apk plugin') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) + .option('--cache-dir ' + , 'where to cache images' + , String + , os.tmpdir()) + .action(function(options) { + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } + + require('./roles/storage/plugins/apk')({ + port: options.port + , storageUrl: options.storageUrl + , cacheDir: options.cacheDir }) }) @@ -523,6 +587,14 @@ program , 'storage port' , Number , 7102) + .option('--storage-plugin-image-port ' + , 'storage image plugin port' + , Number + , 7103) + .option('--storage-plugin-apk-port ' + , 'storage apk plugin port' + , Number + , 7104) .option('--provider ' , 'provider name (or os.hostname())' , String @@ -593,6 +665,8 @@ program , '--connect-push', options.bindDevPull , '--group-timeout', options.groupTimeout , '--public-ip', options.publicIp + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) ].concat(cliutil.allUnknownArgs(args))) // auth-mock @@ -611,6 +685,10 @@ program , '--auth-url', util.format('http://localhost:%d/', options.authPort) , '--storage-url' , util.format('http://localhost:%d/', options.storagePort) + , '--storage-plugin-image-url' + , util.format('http://localhost:%d/', options.storagePluginImagePort) + , '--storage-plugin-apk-url' + , util.format('http://localhost:%d/', options.storagePluginApkPort) , '--connect-sub', options.bindAppPub , '--connect-push', options.bindAppPull ].concat((function() { @@ -625,7 +703,22 @@ program , procutil.fork(__filename, [ 'storage-temp' , '--port', options.storagePort - , '--connect-push', options.bindDevPull + ]) + + // image processor + , procutil.fork(__filename, [ + 'storage-plugin-image' + , '--port', options.storagePluginImagePort + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) + ]) + + // apk processor + , procutil.fork(__filename, [ + 'storage-plugin-apk' + , '--port', options.storagePluginApkPort + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) ]) ] diff --git a/lib/roles/app.js b/lib/roles/app.js index 191e29e6..3ec0f499 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -73,14 +73,22 @@ module.exports = function(options) { , authUrl: options.authUrl })) - // Proxied requests must come before any body parsers - app.post('/api/v1/resources', function(req, res) { + // Proxied requests must come before any body parsers. These proxies are + // here mainly for convenience, they should be replaced with proper reverse + // proxies in production. + app.all('/api/v1/s/image/*', function(req, res) { proxy.web(req, res, { - target: options.storageUrl + target: options.storagePluginImageUrl }) }) - app.get('/api/v1/resources/:id', function(req, res) { + app.all('/api/v1/s/apk/*', function(req, res) { + proxy.web(req, res, { + target: options.storagePluginApkUrl + }) + }) + + app.all('/api/v1/s/*', function(req, res) { proxy.web(req, res, { target: options.storageUrl }) @@ -507,7 +515,11 @@ module.exports = function(options) { channel , wireutil.transaction( responseChannel - , new wire.InstallMessage(data) + , new wire.InstallMessage( + data.href + , data.launch === true + , JSON.stringify(data.manifest) + ) ) ]) }) @@ -637,6 +649,16 @@ module.exports = function(options) { ) ]) }) + .on('screen.capture', function(channel, responseChannel) { + joinChannel(responseChannel) + push.send([ + channel + , wireutil.transaction( + responseChannel + , new wire.ScreenCaptureMessage() + ) + ]) + }) }) .finally(function() { // Clean up all listeners and subscriptions diff --git a/lib/roles/device.js b/lib/roles/device.js index 4f726d97..cace32b8 100644 --- a/lib/roles/device.js +++ b/lib/roles/device.js @@ -19,6 +19,7 @@ module.exports = function(options) { .dependency(require('./device/plugins/solo')) .dependency(require('./device/plugins/heartbeat')) .dependency(require('./device/plugins/display')) + .dependency(require('./device/plugins/screenshot')) .dependency(require('./device/plugins/http')) .dependency(require('./device/plugins/service')) .dependency(require('./device/plugins/browser')) diff --git a/lib/roles/device/plugins/install.js b/lib/roles/device/plugins/install.js index 73283116..71d8b01b 100644 --- a/lib/roles/device/plugins/install.js +++ b/lib/roles/device/plugins/install.js @@ -1,4 +1,6 @@ var stream = require('stream') +var url = require('url') +var util = require('util') var syrup = require('syrup') var request = require('request') @@ -17,7 +19,7 @@ module.exports = syrup.serial() var log = logger.createLogger('device:plugins:install') router.on(wire.InstallMessage, function(channel, message) { - log.info('Installing "%s"', message.url) + log.info('Installing "%s"', message.href) var reply = wireutil.reply(options.serial) @@ -30,7 +32,7 @@ module.exports = syrup.serial() function pushApp() { var req = request({ - url: message.url + url: url.resolve(options.storageUrl, message.href) }) // We need to catch the Content-Length on the fly or we risk @@ -104,16 +106,30 @@ module.exports = syrup.serial() .timeout(30000) }) .then(function() { - if (message.launchActivity) { - log.info( - 'Launching activity with action "%s" on component "%s"' - , message.launchActivity.action - , message.launchActivity.component - ) - // Progress 90% - sendProgress('launching_app', 90) - return adb.startActivity(options.serial, message.launchActivity) - .timeout(15000) + if (message.launch) { + var manifest = JSON.parse(message.manifest) + if (manifest.application.launcherActivities.length) { + var launchActivity = { + action: 'android.intent.action.MAIN' + , component: util.format( + '%s/%s' + , manifest.package + , manifest.application.launcherActivities[0].name + ) + , category: ['android.intent.category.LAUNCHER'] + , flags: 0x10200000 + } + + log.info( + 'Launching activity with action "%s" on component "%s"' + , launchActivity.action + , launchActivity.component + ) + // Progress 90% + sendProgress('launching_app', 90) + return adb.startActivity(options.serial, launchActivity) + .timeout(15000) + } } }) .then(function() { diff --git a/lib/roles/device/plugins/screenshot.js b/lib/roles/device/plugins/screenshot.js new file mode 100644 index 00000000..2b32ac10 --- /dev/null +++ b/lib/roles/device/plugins/screenshot.js @@ -0,0 +1,66 @@ +var http = require('http') +var util = require('util') + +var syrup = require('syrup') +var Promise = require('bluebird') + +var logger = require('../../../util/logger') +var wire = require('../../../wire') +var wireutil = require('../../../wire/util') + +module.exports = syrup.serial() + .dependency(require('../support/router')) + .dependency(require('../support/push')) + .dependency(require('../support/storage')) + .dependency(require('./display')) + .define(function(options, router, push, storage, display) { + var log = logger.createLogger('device:plugins:screenshot') + var plugin = Object.create(null) + + plugin.capture = function() { + log.info('Capturing screenshot from %s', display.url) + + return new Promise(function(resolve, reject) { + var req = http.get(display.url) + + function responseListener(res) { + if (res.statusCode !== 200) { + reject(new Error(util.format( + 'Screenshot capture failed: HTTP %d' + , res.statusCode + ))) + } + else { + resolve(storage.store('image', res, { + filename: util.format('%s.jpg', options.serial) + , contentType: 'image/jpeg' + , knownLength: +res.headers['content-length'] + })) + } + } + + req.on('response', responseListener) + req.on('error', reject) + }) + } + + router.on(wire.ScreenCaptureMessage, function(channel) { + var reply = wireutil.reply(options.serial) + plugin.capture() + .then(function(file) { + push.send([ + channel + , reply.okay('success', file) + ]) + }) + .catch(function(err) { + log.error('Screen capture failed', err.stack) + push.send([ + channel + , reply.fail(err.message) + ]) + }) + }) + + return plugin + }) diff --git a/lib/roles/device/support/storage.js b/lib/roles/device/support/storage.js new file mode 100644 index 00000000..6815c80c --- /dev/null +++ b/lib/roles/device/support/storage.js @@ -0,0 +1,53 @@ +var util = require('util') + +var syrup = require('syrup') +var Promise = require('bluebird') +var request = require('request') + +var logger = require('../../../util/logger') + +module.exports = syrup.serial() + .define(function(options) { + var log = logger.createLogger('device:support:storage') + var plugin = Object.create(null) + + plugin.store = function(type, stream, meta) { + var resolver = Promise.defer() + + var req = request.post({ + url: util.format('%sapi/v1/s/%s', options.storageUrl, type) + } + , function(err, res, body) { + if (err) { + log.error('Upload failed', err.stack) + resolver.reject(err) + } + else if (res.statusCode !== 201) { + log.error('Upload failed: HTTP %d', res.statusCode) + resolver.reject(new Error(util.format( + 'Upload failed: HTTP %d' + , res.statusCode + ))) + } + else { + try { + var result = JSON.parse(body) + log.info('Uploaded to %s', result.resources.file.href) + resolver.resolve(result.resources.file) + } + catch (err) { + log.error('Invalid JSON in response', err.stack, body) + resolver.reject(err) + } + } + } + ) + + req.form() + .append('file', stream, meta) + + return resolver.promise + } + + return plugin + }) diff --git a/lib/roles/storage/plugins/apk/index.js b/lib/roles/storage/plugins/apk/index.js new file mode 100644 index 00000000..6f34c731 --- /dev/null +++ b/lib/roles/storage/plugins/apk/index.js @@ -0,0 +1,54 @@ +var http = require('http') +var url = require('url') + +var express = require('express') +var httpProxy = require('http-proxy') + +var logger = require('../../../../util/logger') +var download = require('../../../../util/download') +var manifest = require('./task/manifest') + +module.exports = function(options) { + var log = logger.createLogger('storage:plugins:apk') + , app = express() + , server = http.createServer(app) + , proxy = httpProxy.createProxyServer() + + proxy.on('error', function(err) { + log.error('Proxy had an error', err.stack) + }) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.get('/api/v1/s/apk/:id/*/manifest', function(req, res) { + download(url.resolve(options.storageUrl, req.url), { + dir: options.cacheDir + }) + .then(manifest) + .then(function(data) { + res.status(200) + .json({ + success: true + , manifest: data + }) + }) + .catch(function(err) { + log.error('Unable to read manifest of "%s"', req.params.id, err.stack) + res.status(500) + .json({ + success: false + }) + }) + }) + + app.get('/api/v1/s/apk/:id/*', function(req, res) { + proxy.web(req, res, { + target: options.storageUrl + }) + }) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/storage/plugins/apk/task/manifest.js b/lib/roles/storage/plugins/apk/task/manifest.js new file mode 100644 index 00000000..6a8f8efb --- /dev/null +++ b/lib/roles/storage/plugins/apk/task/manifest.js @@ -0,0 +1,19 @@ +var Promise = require('bluebird') +var ApkReader = require('adbkit-apkreader') + +module.exports = function(file) { + var resolver = Promise.defer() + + process.nextTick(function() { + try { + var reader = ApkReader.readFile(file.path) + var manifest = reader.readManifestSync() + resolver.resolve(manifest) + } + catch (err) { + resolver.reject(err) + } + }) + + return resolver.promise +} diff --git a/lib/roles/storage/plugins/image/index.js b/lib/roles/storage/plugins/image/index.js new file mode 100644 index 00000000..9fe4d2b9 --- /dev/null +++ b/lib/roles/storage/plugins/image/index.js @@ -0,0 +1,52 @@ +var http = require('http') + +var express = require('express') + +var logger = require('../../../../util/logger') +var requtil = require('../../../../util/requtil') + +var parseCrop = require('./param/crop') +var parseGravity = require('./param/gravity') +var get = require('./task/get') +var transform = require('./task/transform') + +module.exports = function(options) { + var log = logger.createLogger('storage:plugins:image') + , app = express() + , server = http.createServer(app) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.get( + '/api/v1/s/image/:id/*' + , requtil.limit(options.concurrency, function(req, res) { + return get(req.url, options) + .then(function(stream) { + return transform(stream, { + crop: parseCrop(req.query.crop) + , gravity: parseGravity(req.query.gravity) + }) + }) + .then(function(out) { + res.status(200) + out.pipe(res) + }) + .catch(function(err) { + log.error( + 'Unable to transform resource "%s"' + , req.params.id + , err.stack + ) + res.status(500) + .json({ + success: false + }) + }) + }) + ) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/storage/plugins/image/param/crop.js b/lib/roles/storage/plugins/image/param/crop.js new file mode 100644 index 00000000..f719850f --- /dev/null +++ b/lib/roles/storage/plugins/image/param/crop.js @@ -0,0 +1,14 @@ +var RE_CROP = /^([0-9]*)x([0-9]*)$/ + +module.exports = function(raw) { + var parsed + + if (raw && (parsed = RE_CROP.exec(raw))) { + return { + width: +parsed[1] || 0 + , height: +parsed[2] || 0 + } + } + + return null +} diff --git a/lib/roles/storage/plugins/image/param/gravity.js b/lib/roles/storage/plugins/image/param/gravity.js new file mode 100644 index 00000000..bb6ecb6f --- /dev/null +++ b/lib/roles/storage/plugins/image/param/gravity.js @@ -0,0 +1,21 @@ +var GRAVITY = { + northwest: 'NorthWest' +, north: 'North' +, northeast: 'NorthEast' +, west: 'West' +, center: 'Center' +, east: 'East' +, southwest: 'SouthWest' +, south: 'South' +, southeast: 'SouthEast' +} + +module.exports = function(raw) { + var parsed + + if (raw && (parsed = GRAVITY[raw])) { + return parsed + } + + return null +} diff --git a/lib/roles/storage/plugins/image/task/get.js b/lib/roles/storage/plugins/image/task/get.js new file mode 100644 index 00000000..73a12ea2 --- /dev/null +++ b/lib/roles/storage/plugins/image/task/get.js @@ -0,0 +1,20 @@ +var util = require('util') +var http = require('http') +var url = require('url') + +var Promise = require('bluebird') + +module.exports = function(path, options) { + return new Promise(function(resolve, reject) { + http.get(url.resolve(options.storageUrl, path)) + .on('response', function(res) { + if (res.statusCode !== 200) { + reject(new Error(util.format('HTTP %d', res.statusCode))) + } + else { + resolve(res) + } + }) + .on('error', reject) + }) +} diff --git a/lib/roles/storage/plugins/image/task/transform.js b/lib/roles/storage/plugins/image/task/transform.js new file mode 100644 index 00000000..b87ccbbe --- /dev/null +++ b/lib/roles/storage/plugins/image/task/transform.js @@ -0,0 +1,26 @@ +var gm = require('gm') +var Promise = require('bluebird') + +module.exports = function(stream, options) { + return new Promise(function(resolve, reject) { + var transform = gm(stream) + + if (options.gravity) { + transform.gravity(options.gravity) + } + + if (options.crop) { + transform.geometry(options.crop.width, options.crop.height, '^') + transform.crop(options.crop.width, options.crop.height, 0, 0) + } + + transform.stream(function(err, stdout) { + if (err) { + reject(err) + } + else { + resolve(stdout) + } + }) + }) +} diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index 1bd81f47..3681cc06 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -1,35 +1,23 @@ var http = require('http') var util = require('util') -var fs = require('fs') +var path = require('path') var express = require('express') var validator = require('express-validator') var formidable = require('formidable') var Promise = require('bluebird') -var ApkReader = require('adbkit-apkreader') -var request = require('request') -var progress = require('request-progress') -var temp = require('temp') -var zmq = require('zmq') var logger = require('../../util/logger') -var requtil = require('../../util/requtil') var Storage = require('../../util/storage') -var wireutil = require('../../wire/util') +var requtil = require('../../util/requtil') +var download = require('../../util/download') module.exports = function(options) { - var log = logger.createLogger('storage-temp') + var log = logger.createLogger('storage:temp') , app = express() , server = http.createServer(app) , storage = new Storage() - // Output - var push = zmq.socket('push') - options.endpoints.push.forEach(function(endpoint) { - log.info('Sending output to %s', endpoint) - push.connect(endpoint) - }) - app.set('strict routing', true) app.set('case sensitive routing', true) app.set('trust proxy', true) @@ -41,175 +29,42 @@ module.exports = function(options) { log.info('Cleaning up inactive resource "%s"', id) }) - function processFile(file) { - var resolver = Promise.defer() - - log.info('Processing file "%s"', file.path) - - resolver.progress({ - percent: 0 - }) - - process.nextTick(function() { - try { - var reader = ApkReader.readFile(file.path) - var manifest = reader.readManifestSync() - resolver.resolve(manifest) - } - catch (err) { - err.reportCode = 'fail_invalid_app_file' - resolver.reject(err) - } - }) - - return resolver.promise - } - - function storeFile(file) { - var id = storage.store(file) - return Promise.resolve({ - id: id - , url: util.format( - 'http://%s:%s/api/v1/resources/%s' - , options.publicIp - , options.port - , id - ) - }) - } - - function download(url) { - var resolver = Promise.defer() - var path = temp.path({ - dir: options.saveDir - }) - - log.info('Downloading "%s" to "%s"', url, path) - - function errorListener(err) { - err.reportCode = 'fail_download' - resolver.reject(err) - } - - function progressListener(state) { - resolver.progress(state) - } - - function closeListener() { - resolver.resolve({ - path: path - }) - } - - resolver.progress({ - percent: 0 - }) - - try { - var req = progress(request(url), { - throttle: 100 // Throttle events, not upload speed - }) - .on('progress', progressListener) - - var save = req.pipe(fs.createWriteStream(path)) - .on('error', errorListener) - .on('close', closeListener) - } - catch (err) { - err.reportCode = 'fail_invalid_url' - resolver.reject(err) - } - - return resolver.promise.finally(function() { - req.removeListener('progress', progressListener) - save.removeListener('error', errorListener) - save.removeListener('close', closeListener) - }) - } - - app.post('/api/v1/resources', function(req, res) { - var reply = wireutil.reply(options.id) - - function sendProgress(data, progress) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.progress(data, progress) - ]) - } - } - - function sendDone(success, data, body) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.okay(data, body) - ]) - } - } - + app.post('/api/v1/s/:type/download', function(req, res) { requtil.validate(req, function() { - req.checkQuery('channel').notEmpty() + req.checkBody('url').notEmpty() }) .then(function() { - if (req.is('application/json')) { - return requtil.validate(req, function() { - req.checkBody('url').notEmpty() - }) - .then(function() { - return download(req.body.url) - .progressed(function(progress) { - sendProgress('uploading', 0.7 * progress.percent) - }) - }) - } - else { - var form = Promise.promisifyAll(new formidable.IncomingForm()) - var progressListener = function(received, expected) { - if (expected) { - sendProgress('uploading', 70 * (received / expected)) - } - } - sendProgress('uploading', 0) - form.on('progress', progressListener) - return form.parseAsync(req) - .finally(function() { - form.removeListener('progress', progressListener) - }) - .spread(function(fields, files) { - if (!files.file) { - throw new requtil.ValidationError('validation error', [ - { - "param": "file" - , "msg": "Required value" - } - ]) - } - return files.file - }) + return download(req.body.url, { + dir: options.cacheDir + }) + }) + .then(function(file) { + return { + id: storage.store(file) + , name: file.name } }) .then(function(file) { - return processFile(file) - .progressed(function(progress) { - sendProgress('processing', 70 + 0.2 * progress.percent) + res.status(201) + .json({ + success: true + , resource: { + date: new Date() + , type: req.params.type + , id: file.id + , name: file.name + , href: util.format( + '/api/v1/s/%s/%s%s' + , req.params.type + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) + } }) - .then(function(manifest) { - sendProgress('storing', 90) - return storeFile(file) - .then(function(data) { - data.manifest = manifest - return data - }) - }) - }) - .then(function(data) { - sendDone(true, 'success', data) - data.success = true - res.json(201, data) }) .catch(requtil.ValidationError, function(err) { - sendDone(false, err.reportCode || 'fail_validation') res.status(400) .json({ success: false @@ -218,8 +73,7 @@ module.exports = function(options) { }) }) .catch(function(err) { - log.error('Unexpected error', err.stack) - sendDone(false, err.reportCode || 'fail') + log.error('Error storing resource', err.stack) res.status(500) .json({ success: false @@ -228,7 +82,57 @@ module.exports = function(options) { }) }) - app.get('/api/v1/resources/:id', function(req, res) { + app.post('/api/v1/s/:type', function(req, res) { + var form = new formidable.IncomingForm() + Promise.promisify(form.parse, form)(req) + .spread(function(fields, files) { + return Object.keys(files).map(function(field) { + var file = files[field] + log.info('Uploaded "%s" to "%s"', file.name, file.path) + return { + field: field + , id: storage.store(file) + , name: file.name + } + }) + }) + .then(function(storedFiles) { + res.status(201) + .json({ + success: true + , resources: (function() { + var mapped = Object.create(null) + storedFiles.forEach(function(file) { + mapped[file.field] = { + date: new Date() + , type: req.params.type + , id: file.id + , name: file.name + , href: util.format( + '/api/v1/s/%s/%s%s' + , req.params.type + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) + } + }) + return mapped + })() + }) + }) + .catch(function(err) { + log.error('Error storing resource', err.stack) + res.status(500) + .json({ + success: false + , error: 'ServerError' + }) + }) + }) + + app.get('/api/v1/s/:type/:id/*', function(req, res) { var file = storage.retrieve(req.params.id) if (file) { res.set('Content-Type', file.type) diff --git a/lib/util/download.js b/lib/util/download.js new file mode 100644 index 00000000..372345e8 --- /dev/null +++ b/lib/util/download.js @@ -0,0 +1,67 @@ +var fs = require('fs') + +var Promise = require('bluebird') +var request = require('request') +var progress = require('request-progress') +var temp = require('temp') + +module.exports = function download(url, options) { + var resolver = Promise.defer() + var path = temp.path(options) + + function errorListener(err) { + resolver.reject(err) + } + + function progressListener(state) { + if (state.total !== null) { + resolver.progress({ + lengthComputable: true + , loaded: state.received + , total: state.total + }) + } + else { + resolver.progress({ + lengthComputable: false + , loaded: state.received + , total: state.received + }) + } + } + + function closeListener() { + resolver.resolve({ + path: path + }) + } + + resolver.progress({ + percent: 0 + }) + + try { + var req = progress(request(url), { + throttle: 100 // Throttle events, not upload speed + }) + .on('progress', progressListener) + + resolver.promise.finally(function() { + req.removeListener('progress', progressListener) + }) + + var save = req.pipe(fs.createWriteStream(path)) + .on('error', errorListener) + .on('close', closeListener) + + resolver.promise.finally(function() { + save.removeListener('error', errorListener) + save.removeListener('close', closeListener) + }) + } + catch (err) { + resolver.reject(err) + } + + return resolver.promise +} diff --git a/lib/util/requtil.js b/lib/util/requtil.js index 35390962..79d85210 100644 --- a/lib/util/requtil.js +++ b/lib/util/requtil.js @@ -26,3 +26,25 @@ module.exports.validate = function(req, rules) { } }) } + +module.exports.limit = function(limit, handler) { + var queue = [] + var running = 0 + + function done() { + running -= 1 + maybeNext() + } + + function maybeNext() { + while (running < limit && queue.length) { + running += 1 + handler.apply(null, queue.shift()).finally(done) + } + } + + return function() { + queue.push(arguments) + maybeNext() + } +} diff --git a/lib/util/storage.js b/lib/util/storage.js index 3ea23ebf..ec8fd7d1 100644 --- a/lib/util/storage.js +++ b/lib/util/storage.js @@ -14,14 +14,18 @@ util.inherits(Storage, events.EventEmitter) Storage.prototype.store = function(file) { var id = uuid.v4() + this.set(id, file) + return id +} +Storage.prototype.set = function(id, file) { this.files[id] = { timeout: 600000 , lastActivity: Date.now() , data: file } - return id + return file } Storage.prototype.remove = function(id) { diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 5a3146d4..eb5971e2 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -50,6 +50,7 @@ enum MessageType { PhoneStateEvent = 47; RotationEvent = 48; StoreOpenMessage = 49; + ScreenCaptureMessage = 50; } message Envelope { @@ -353,8 +354,9 @@ message ShellKeepAliveMessage { } message InstallMessage { - required string url = 1; - optional LaunchActivityMessage launchActivity = 2; + required string href = 1; + required bool launch = 2; + optional string manifest = 3; } message UninstallMessage { @@ -400,6 +402,9 @@ message BrowserClearMessage { message StoreOpenMessage { } +message ScreenCaptureMessage { +} + // Events, these must be kept in sync with STFService/wire.proto message AirplaneModeEvent { diff --git a/res/app/app.js b/res/app/app.js index c36e03e0..ec9d0a66 100644 --- a/res/app/app.js +++ b/res/app/app.js @@ -3,7 +3,8 @@ require('angular-route') require('angular-touch') require('angular-gettext') -require('ng-file-upload') +require('ng-file-upload-shim5') +require('ng-file-upload-main') angular.module('app', [ 'ngRoute', diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index c9bbdd9d..5f9661e9 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -111,49 +111,8 @@ module.exports = function ControlServiceFactory( return sendTwoWay('device.identify') } - this.uploadUrl = function(url) { - var tx = TransactionService.create({ - id: 'storage' - }) - socket.emit('storage.upload', channel, tx.channel, { - url: url - }) - return tx.promise - } - - this.uploadFile = function(files) { - if (files.length !== 1) { - throw new Error('Can only upload one file') - } - var tx = TransactionService.create({ - id: 'storage' - }) - TransactionService.punch(tx.channel) - .then(function() { - $upload.upload({ - url: '/api/v1/resources?channel=' + tx.channel - , method: 'POST' - , file: files[0] - }) - }) - return tx.promise - } - this.install = function(options) { - var app = options.manifest.application - var params = { - url: options.url - } - if (app.launcherActivities.length) { - var activity = app.launcherActivities[0] - params.launchActivity = { - action: 'android.intent.action.MAIN' - , component: options.manifest.package + '/' + activity.name - , category: ['android.intent.category.LAUNCHER'] - , flags: 0x10200000 - } - } - return sendTwoWay('device.install', params) + return sendTwoWay('device.install', options) } this.uninstall = function(pkg) { @@ -216,6 +175,10 @@ module.exports = function ControlServiceFactory( return sendTwoWay('store.open') } + this.screenshot = function() { + return sendTwoWay('screen.capture') + } + window.cc = this } diff --git a/res/app/components/stf/control/index.js b/res/app/components/stf/control/index.js index 5e5d01b3..5644cffa 100644 --- a/res/app/components/stf/control/index.js +++ b/res/app/components/stf/control/index.js @@ -3,3 +3,4 @@ module.exports = angular.module('stf/control', [ ]) .factory('TransactionService', require('./transaction-service')) .factory('ControlService', require('./control-service')) + .factory('StorageService', require('./storage-service')) diff --git a/res/app/components/stf/control/storage-service.js b/res/app/components/stf/control/storage-service.js new file mode 100644 index 00000000..bbc3d287 --- /dev/null +++ b/res/app/components/stf/control/storage-service.js @@ -0,0 +1,40 @@ +var Promise = require('bluebird') + +module.exports = function StorageServiceFactory($http, $upload) { + var service = {} + + service.storeUrl = function(type, url) { + return $http({ + url: '/api/v1/s/' + type + '/download' + , method: 'POST' + , data: { + url: url + } + }) + } + + service.storeFile = function(type, files) { + var resolver = Promise.defer() + + $upload.upload({ + url: '/api/v1/s/' + type + , method: 'POST' + , file: files + }) + .then( + function(value) { + resolver.resolve(value) + } + , function(err) { + resolver.reject(err) + } + , function(progressEvent) { + resolver.progress(progressEvent) + } + ) + + return resolver.promise + } + + return service +} diff --git a/res/app/control-panes/dashboard/upload/upload-controller.js b/res/app/control-panes/dashboard/upload/upload-controller.js index e0e87584..4407f5cb 100644 --- a/res/app/control-panes/dashboard/upload/upload-controller.js +++ b/res/app/control-panes/dashboard/upload/upload-controller.js @@ -1,5 +1,9 @@ -module.exports = function UploadCtrl($scope, SettingsService, gettext) { - +module.exports = function UploadCtrl( + $scope +, $http +, SettingsService +, StorageService +) { $scope.upload = null $scope.installation = null $scope.installEnabled = true @@ -35,23 +39,49 @@ module.exports = function UploadCtrl($scope, SettingsService, gettext) { $scope.installFile = function ($files) { $scope.upload = { - progress: 0, - lastData: 'uploading' + progress: 0 + , lastData: 'uploading' } - $scope.installation = null - return $scope.control.uploadFile($files) - .progressed(function (uploadResult) { - $scope.$apply(function () { - $scope.upload = uploadResult - }) + return StorageService.storeFile('apk', $files) + .progressed(function(e) { + if (e.lengthComputable) { + $scope.upload = { + progress: e.loaded / e.total * 100 + , lastData: 'uploading' + } + } }) - .then(function (uploadResult) { - $scope.$apply(function () { - $scope.upload = uploadResult - }) - if (uploadResult.success) { - return $scope.maybeInstall(uploadResult.body) + .then(function(res) { + $scope.upload = { + progress: 100 + , lastData: 'processing' + } + + var href = res.data.resources.file0.href + return $http.get(href + '/manifest') + .then(function(res) { + $scope.upload = { + progress: 100 + , lastData: 'success' + , settled: true + } + + if (res.data.success) { + return $scope.maybeInstall({ + href: href + , launch: $scope.launchEnabled + , manifest: res.data.manifest + }) + } + }) + }) + .catch(function(err) { + console.log('Upload error', err) + $scope.upload = { + progress: 100 + , lastData: 'fail' + , settled: true } }) } diff --git a/res/app/control-panes/dashboard/upload/upload.jade b/res/app/control-panes/dashboard/upload/upload.jade index 50f4dae3..1b9cc953 100644 --- a/res/app/control-panes/dashboard/upload/upload.jade +++ b/res/app/control-panes/dashboard/upload/upload.jade @@ -3,9 +3,9 @@ i.fa.fa-upload span(translate) Upload clear-button(ng-click='clear()', ng-disabled='!installation && !upload').btn-xs - //label.checkbox-inline.pull-right - input(type='checkbox', ng-model='launchEnabled', ng-disabled='true') - span(translate) Launch + label.checkbox-inline.pull-right + input(type='checkbox', ng-model='launchEnabled') + span Launch label.checkbox-inline.pull-right input(type='checkbox', ng-model='installEnabled') span(translate) Install @@ -65,8 +65,6 @@ span(translate) Uploading... strong(ng-switch-when='processing') span(translate) Processing... - strong(ng-switch-when='storing') - span(translate) Storing... strong(ng-switch-when='fail') span(translate) Upload failed strong(ng-switch-when='success') diff --git a/res/app/control-panes/screenshots/screenshots-controller.js b/res/app/control-panes/screenshots/screenshots-controller.js index 71b51eef..89e4de1a 100644 --- a/res/app/control-panes/screenshots/screenshots-controller.js +++ b/res/app/control-panes/screenshots/screenshots-controller.js @@ -1,3 +1,31 @@ -module.exports = function ScreenshotsCtrl($scope) { - +module.exports = function ScreenshotsCtrl($scope, SettingsService) { + $scope.screenshots = [] + $scope.shotSize = 'small' + + $scope.clear = function () { + $scope.screenshots = [] + } + + SettingsService.bind($scope, { + key: 'shotSize' + , storeName: 'ScreenShots.shotSize' + }) + + $scope.shotSizeUrlParameter = function () { + var sizes = { + 'small': '?crop=100x', + 'medium': '?crop=320x', + 'large': '?crop=450x', + 'original': '' + } + return sizes[$scope.shotSize] + } + + $scope.takeScreenShot = function () { + $scope.control.screenshot().then(function(result) { + $scope.$apply(function() { + $scope.screenshots.push(result) + }) + }) + } } diff --git a/res/app/control-panes/screenshots/screenshots.jade b/res/app/control-panes/screenshots/screenshots.jade index aff7c630..832eefcb 100644 --- a/res/app/control-panes/screenshots/screenshots.jade +++ b/res/app/control-panes/screenshots/screenshots.jade @@ -24,9 +24,8 @@ nothing-to-show(message='{{"No screenshots taken"|translate}}', icon='fa-camera', ng-show='!screenshots.length') ul.screenshots-icon-view.clear-fix li(ng-repeat='shot in screenshots').cursor-select - h4 {{ shot.device.capabilities.info.name.id || shot.device.capabilities.model }} - h5 {{ shot.value.date | date:'yyyy/MM/dd HH:mm:ss' }} - a(ng-href='{{ shot.value.screenshotUrl }}', target='_blank') - img(ng-src='{{ shot.value.screenshotUrl + shotSizeUrlParameter() }}') + h5 {{ shot.body.date | date:'yyyy/MM/dd HH:mm:ss' }} + a(ng-href='{{ shot.body.href }}', target='_blank') + img(ng-src='{{ shot.body.href + shotSizeUrlParameter() }}') - .clearfix \ No newline at end of file + .clearfix diff --git a/webpack.config.js b/webpack.config.js index 67778886..3935dab0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -26,7 +26,8 @@ module.exports = { , 'localforage': 'localforage/dist/localforage.js' , 'socket.io': 'socket.io-client/dist/socket.io' , 'oboe': 'oboe/dist/oboe-browser' - , 'ng-file-upload': 'ng-file-upload/angular-file-upload' + , 'ng-file-upload-shim5': 'ng-file-upload/angular-file-upload-html5-shim' + , 'ng-file-upload-main': 'ng-file-upload/angular-file-upload' , 'bluebird': 'bluebird/js/browser/bluebird' } }