From 48726669dc20116c1b3f9145128587aaaddf2893 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 19 May 2014 10:25:13 +0900 Subject: [PATCH 1/9] Screenshots are more or less working (server-side). Resize not implemented yet. Temporarily breaks APK uploads. --- lib/cli.js | 44 +++- lib/roles/app.js | 10 + lib/roles/cache/apk.js | 216 +++++++++++++++++ lib/roles/device.js | 1 + lib/roles/device/plugins/screenshot.js | 66 ++++++ lib/roles/device/support/storage.js | 53 +++++ lib/roles/storage/temp.js | 222 ++++-------------- lib/wire/wire.proto | 4 + .../components/stf/control/control-service.js | 4 + 9 files changed, 435 insertions(+), 185 deletions(-) create mode 100644 lib/roles/cache/apk.js create mode 100644 lib/roles/device/plugins/screenshot.js create mode 100644 lib/roles/device/support/storage.js diff --git a/lib/cli.js b/lib/cli.js index 569d71d4..32e92ea0 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -43,6 +43,9 @@ program , 'group timeout' , Number , 600) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) .action(function() { var serials = cliutil.allUnknownArgs(arguments) , options = cliutil.lastArg(arguments) @@ -53,6 +56,9 @@ program if (!options.connectPush) { this.missingArgument('--connect-push') } + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } require('./roles/provider')({ name: options.name @@ -71,6 +77,7 @@ program , '--ports', ports.join(',') , '--public-ip', options.publicIp , '--group-timeout', options.groupTimeout + , '--storage-url', options.storageUrl ]) } , endpoints: { @@ -107,6 +114,9 @@ program , 'group timeout' , Number , 600) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) .action(function(serial, options) { if (!options.connectSub) { this.missingArgument('--connect-sub') @@ -120,6 +130,9 @@ program if (!options.ports) { this.missingArgument('--ports') } + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } require('./roles/device')({ serial: serial @@ -132,6 +145,7 @@ program } , heartbeatInterval: options.heartbeatInterval , groupTimeout: options.groupTimeout * 1000 // change to ms + , storageUrl: options.storageUrl }) }) @@ -426,8 +440,8 @@ program }) program - .command('storage-temp') - .description('start temp storage') + .command('cache-apk') + .description('apk cache') .option('-p, --port ' , 'port (or $PORT)' , Number @@ -463,6 +477,29 @@ program }) }) +program + .command('storage-temp') + .description('start temp storage') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('--public-ip ' + , 'public ip for global access' + , String + , ip()) + .option('--save-dir ' + , 'where to save files' + , String + , os.tmpdir()) + .action(function(options) { + require('./roles/storage/temp')({ + port: options.port + , publicIp: options.publicIp + , saveDir: options.saveDir + }) + }) + program .command('migrate') .description('migrates the database to the latest version') @@ -593,6 +630,8 @@ program , '--connect-push', options.bindDevPull , '--group-timeout', options.groupTimeout , '--public-ip', options.publicIp + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) ].concat(cliutil.allUnknownArgs(args))) // auth-mock @@ -625,7 +664,6 @@ program , procutil.fork(__filename, [ 'storage-temp' , '--port', options.storagePort - , '--connect-push', options.bindDevPull ]) ] diff --git a/lib/roles/app.js b/lib/roles/app.js index 191e29e6..c8642d5a 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -637,6 +637,16 @@ module.exports = function(options) { ) ]) }) + .on('screen.capture', function(channel, responseChannel) { + joinChannel(responseChannel) + push.send([ + channel + , wireutil.transaction( + responseChannel + , new wire.ScreenCaptureMessage() + ) + ]) + }) }) .finally(function() { // Clean up all listeners and subscriptions diff --git a/lib/roles/cache/apk.js b/lib/roles/cache/apk.js new file mode 100644 index 00000000..718dcb15 --- /dev/null +++ b/lib/roles/cache/apk.js @@ -0,0 +1,216 @@ +var http = require('http') +var util = require('util') +var fs = require('fs') + +var express = require('express') +var validator = require('express-validator') +var Promise = require('bluebird') +var ApkReader = require('adbkit-apkreader') +var request = require('request') +var progress = require('request-progress') +var temp = require('temp') +var zmq = require('zmq') + +var logger = require('../../util/logger') +var requtil = require('../../util/requtil') +var Storage = require('../../util/storage') +var wireutil = require('../../wire/util') + +module.exports = function(options) { + var log = logger.createLogger('cache-apk') + , app = express() + , server = http.createServer(app) + , storage = new Storage() + + // Output + var push = zmq.socket('push') + options.endpoints.push.forEach(function(endpoint) { + log.info('Sending output to %s', endpoint) + push.connect(endpoint) + }) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.use(express.json()) + app.use(validator()) + + storage.on('timeout', function(id) { + log.info('Cleaning up inactive resource "%s"', id) + }) + + function processFile(file) { + var resolver = Promise.defer() + + log.info('Processing file "%s"', file.path) + + resolver.progress({ + percent: 0 + }) + + process.nextTick(function() { + try { + var reader = ApkReader.readFile(file.path) + var manifest = reader.readManifestSync() + resolver.resolve(manifest) + } + catch (err) { + err.reportCode = 'fail_invalid_app_file' + resolver.reject(err) + } + }) + + return resolver.promise + } + + function storeFile(file) { + var id = storage.store(file) + return Promise.resolve({ + id: id + , url: util.format( + 'http://%s:%s/api/v1/resources/%s' + , options.publicIp + , options.port + , id + ) + }) + } + + function download(url) { + var resolver = Promise.defer() + var path = temp.path({ + dir: options.saveDir + }) + + log.info('Downloading "%s" to "%s"', url, path) + + function errorListener(err) { + err.reportCode = 'fail_download' + resolver.reject(err) + } + + function progressListener(state) { + resolver.progress(state) + } + + function closeListener() { + resolver.resolve({ + path: path + }) + } + + resolver.progress({ + percent: 0 + }) + + try { + var req = progress(request(url), { + throttle: 100 // Throttle events, not upload speed + }) + .on('progress', progressListener) + + var save = req.pipe(fs.createWriteStream(path)) + .on('error', errorListener) + .on('close', closeListener) + } + catch (err) { + err.reportCode = 'fail_invalid_url' + resolver.reject(err) + } + + return resolver.promise.finally(function() { + req.removeListener('progress', progressListener) + save.removeListener('error', errorListener) + save.removeListener('close', closeListener) + }) + } + + app.post('/api/v1/cache', function(req, res) { + var reply = wireutil.reply(options.id) + + function sendProgress(data, progress) { + if (req.query.channel) { + push.send([ + req.query.channel + , reply.progress(data, progress) + ]) + } + } + + function sendDone(success, data, body) { + if (req.query.channel) { + push.send([ + req.query.channel + , reply.okay(data, body) + ]) + } + } + + requtil.validate(req, function() { + req.checkQuery('channel').notEmpty() + }) + .then(function() { + return requtil.validate(req, function() { + req.checkBody('url').notEmpty() + }) + .then(function() { + return download(req.body.url) + .progressed(function(progress) { + sendProgress('uploading', 0.7 * progress.percent) + }) + }) + }) + .then(function(file) { + return processFile(file) + .progressed(function(progress) { + sendProgress('processing', 70 + 0.2 * progress.percent) + }) + .then(function(manifest) { + sendProgress('storing', 90) + return storeFile(file) + .then(function(data) { + data.manifest = manifest + return data + }) + }) + }) + .then(function(data) { + sendDone(true, 'success', data) + data.success = true + res.json(201, data) + }) + .catch(requtil.ValidationError, function(err) { + sendDone(false, err.reportCode || 'fail_validation') + res.status(400) + .json({ + success: false + , error: 'ValidationError' + , validationErrors: err.errors + }) + }) + .catch(function(err) { + log.error('Unexpected error', err.stack) + sendDone(false, err.reportCode || 'fail') + res.status(500) + .json({ + success: false + , error: 'ServerError' + }) + }) + }) + + app.get('/api/v1/cache/:id', function(req, res) { + var file = storage.retrieve(req.params.id) + if (file) { + res.set('Content-Type', file.type) + res.sendfile(file.path) + } + else { + res.send(404) + } + }) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/device.js b/lib/roles/device.js index 4f726d97..cace32b8 100644 --- a/lib/roles/device.js +++ b/lib/roles/device.js @@ -19,6 +19,7 @@ module.exports = function(options) { .dependency(require('./device/plugins/solo')) .dependency(require('./device/plugins/heartbeat')) .dependency(require('./device/plugins/display')) + .dependency(require('./device/plugins/screenshot')) .dependency(require('./device/plugins/http')) .dependency(require('./device/plugins/service')) .dependency(require('./device/plugins/browser')) diff --git a/lib/roles/device/plugins/screenshot.js b/lib/roles/device/plugins/screenshot.js new file mode 100644 index 00000000..1dd2e97a --- /dev/null +++ b/lib/roles/device/plugins/screenshot.js @@ -0,0 +1,66 @@ +var http = require('http') +var util = require('util') + +var syrup = require('syrup') +var Promise = require('bluebird') + +var logger = require('../../../util/logger') +var wire = require('../../../wire') +var wireutil = require('../../../wire/util') + +module.exports = syrup.serial() + .dependency(require('../support/router')) + .dependency(require('../support/push')) + .dependency(require('../support/storage')) + .dependency(require('./display')) + .define(function(options, router, push, storage, display) { + var log = logger.createLogger('device:plugins:screenshot') + var plugin = Object.create(null) + + plugin.capture = function() { + log.info('Capturing screenshot from %s', display.url) + + return new Promise(function(resolve, reject) { + var req = http.get(display.url) + + function responseListener(res) { + if (res.statusCode !== 200) { + reject(new Error(util.format( + 'Screenshot capture failed: HTTP %d' + , res.statusCode + ))) + } + else { + resolve(storage.store(res, { + filename: util.format('%s.png', options.serial) + , contentType: 'image/png' + , knownLength: +res.headers['content-length'] + })) + } + } + + req.on('response', responseListener) + req.on('error', reject) + }) + } + + router.on(wire.ScreenCaptureMessage, function(channel) { + var reply = wireutil.reply(options.serial) + plugin.capture() + .then(function(url) { + push.send([ + channel + , reply.okay(url) + ]) + }) + .catch(function(err) { + log.error('Screen capture failed', err.stack) + push.send([ + channel + , reply.fail(err.message) + ]) + }) + }) + + return plugin + }) diff --git a/lib/roles/device/support/storage.js b/lib/roles/device/support/storage.js new file mode 100644 index 00000000..7009222b --- /dev/null +++ b/lib/roles/device/support/storage.js @@ -0,0 +1,53 @@ +var util = require('util') + +var syrup = require('syrup') +var Promise = require('bluebird') +var request = require('request') + +var logger = require('../../../util/logger') + +module.exports = syrup.serial() + .define(function(options) { + var log = logger.createLogger('device:support:storage') + var plugin = Object.create(null) + + plugin.store = function(stream, meta) { + var resolver = Promise.defer() + + var req = request.post({ + url: util.format('%sapi/v1/resources', options.storageUrl) + } + , function(err, res, body) { + if (err) { + log.error('Upload failed', err.stack) + resolver.reject(err) + } + else if (res.statusCode !== 201) { + log.error('Upload failed: HTTP %d', res.statusCode) + resolver.reject(new Error(util.format( + 'Upload failed: HTTP %d' + , res.statusCode + ))) + } + else { + try { + var result = JSON.parse(body) + log.info('Uploaded to %s', result.resources.file) + resolver.resolve(result.resources.file) + } + catch (err) { + log.error('Invalid JSON in response', err.stack, body) + resolver.reject(err) + } + } + } + ) + + req.form() + .append('file', stream, meta) + + return resolver.promise + } + + return plugin + }) diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index 1bd81f47..2454793e 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -1,21 +1,14 @@ var http = require('http') var util = require('util') -var fs = require('fs') +var path = require('path') var express = require('express') -var validator = require('express-validator') var formidable = require('formidable') var Promise = require('bluebird') -var ApkReader = require('adbkit-apkreader') -var request = require('request') -var progress = require('request-progress') -var temp = require('temp') -var zmq = require('zmq') var logger = require('../../util/logger') var requtil = require('../../util/requtil') var Storage = require('../../util/storage') -var wireutil = require('../../wire/util') module.exports = function(options) { var log = logger.createLogger('storage-temp') @@ -23,193 +16,48 @@ module.exports = function(options) { , server = http.createServer(app) , storage = new Storage() - // Output - var push = zmq.socket('push') - options.endpoints.push.forEach(function(endpoint) { - log.info('Sending output to %s', endpoint) - push.connect(endpoint) - }) - app.set('strict routing', true) app.set('case sensitive routing', true) app.set('trust proxy', true) - app.use(express.json()) - app.use(validator()) - storage.on('timeout', function(id) { log.info('Cleaning up inactive resource "%s"', id) }) - function processFile(file) { - var resolver = Promise.defer() - - log.info('Processing file "%s"', file.path) - - resolver.progress({ - percent: 0 - }) - - process.nextTick(function() { - try { - var reader = ApkReader.readFile(file.path) - var manifest = reader.readManifestSync() - resolver.resolve(manifest) - } - catch (err) { - err.reportCode = 'fail_invalid_app_file' - resolver.reject(err) - } - }) - - return resolver.promise - } - - function storeFile(file) { - var id = storage.store(file) - return Promise.resolve({ - id: id - , url: util.format( - 'http://%s:%s/api/v1/resources/%s' - , options.publicIp - , options.port - , id - ) - }) - } - - function download(url) { - var resolver = Promise.defer() - var path = temp.path({ - dir: options.saveDir - }) - - log.info('Downloading "%s" to "%s"', url, path) - - function errorListener(err) { - err.reportCode = 'fail_download' - resolver.reject(err) - } - - function progressListener(state) { - resolver.progress(state) - } - - function closeListener() { - resolver.resolve({ - path: path - }) - } - - resolver.progress({ - percent: 0 - }) - - try { - var req = progress(request(url), { - throttle: 100 // Throttle events, not upload speed - }) - .on('progress', progressListener) - - var save = req.pipe(fs.createWriteStream(path)) - .on('error', errorListener) - .on('close', closeListener) - } - catch (err) { - err.reportCode = 'fail_invalid_url' - resolver.reject(err) - } - - return resolver.promise.finally(function() { - req.removeListener('progress', progressListener) - save.removeListener('error', errorListener) - save.removeListener('close', closeListener) - }) - } - app.post('/api/v1/resources', function(req, res) { - var reply = wireutil.reply(options.id) - - function sendProgress(data, progress) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.progress(data, progress) - ]) - } - } - - function sendDone(success, data, body) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.okay(data, body) - ]) - } - } - - requtil.validate(req, function() { - req.checkQuery('channel').notEmpty() - }) - .then(function() { - if (req.is('application/json')) { - return requtil.validate(req, function() { - req.checkBody('url').notEmpty() - }) - .then(function() { - return download(req.body.url) - .progressed(function(progress) { - sendProgress('uploading', 0.7 * progress.percent) - }) - }) - } - else { - var form = Promise.promisifyAll(new formidable.IncomingForm()) - var progressListener = function(received, expected) { - if (expected) { - sendProgress('uploading', 70 * (received / expected)) - } + var form = new formidable.IncomingForm() + Promise.promisify(form.parse, form)(req) + .spread(function(fields, files) { + return Object.keys(files).map(function(field) { + return { + field: field + , id: storage.store(files[field]) + , name: files[field].name } - sendProgress('uploading', 0) - form.on('progress', progressListener) - return form.parseAsync(req) - .finally(function() { - form.removeListener('progress', progressListener) - }) - .spread(function(fields, files) { - if (!files.file) { - throw new requtil.ValidationError('validation error', [ - { - "param": "file" - , "msg": "Required value" - } - ]) - } - return files.file - }) - } + }) }) - .then(function(file) { - return processFile(file) - .progressed(function(progress) { - sendProgress('processing', 70 + 0.2 * progress.percent) - }) - .then(function(manifest) { - sendProgress('storing', 90) - return storeFile(file) - .then(function(data) { - data.manifest = manifest - return data + .then(function(storedFiles) { + res.status(201) + .json({ + success: true + , resources: (function() { + var mapped = Object.create(null) + storedFiles.forEach(function(file) { + mapped[file.field] = util.format( + 'http://%s:%s/api/v1/resources/%s%s' + , options.publicIp + , options.port + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) }) + return mapped + })() }) }) - .then(function(data) { - sendDone(true, 'success', data) - data.success = true - res.json(201, data) - }) .catch(requtil.ValidationError, function(err) { - sendDone(false, err.reportCode || 'fail_validation') res.status(400) .json({ success: false @@ -218,8 +66,7 @@ module.exports = function(options) { }) }) .catch(function(err) { - log.error('Unexpected error', err.stack) - sendDone(false, err.reportCode || 'fail') + log.error('Error storing resource', err.stack) res.status(500) .json({ success: false @@ -239,6 +86,17 @@ module.exports = function(options) { } }) + app.get('/api/v1/resources/:id/*', function(req, res) { + var file = storage.retrieve(req.params.id) + if (file) { + res.set('Content-Type', file.type) + res.sendfile(file.path) + } + else { + res.send(404) + } + }) + server.listen(options.port) log.info('Listening on port %d', options.port) } diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 5a3146d4..f8e67fea 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -50,6 +50,7 @@ enum MessageType { PhoneStateEvent = 47; RotationEvent = 48; StoreOpenMessage = 49; + ScreenCaptureMessage = 50; } message Envelope { @@ -400,6 +401,9 @@ message BrowserClearMessage { message StoreOpenMessage { } +message ScreenCaptureMessage { +} + // Events, these must be kept in sync with STFService/wire.proto message AirplaneModeEvent { diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index c9bbdd9d..40f87f3f 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -216,6 +216,10 @@ module.exports = function ControlServiceFactory( return sendTwoWay('store.open') } + this.screenshot = function() { + return sendTwoWay('screen.capture') + } + window.cc = this } From 254b2dc7a3ba5ff2bf607166a8edb5e8a2c18756 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 20 May 2014 11:34:39 +0900 Subject: [PATCH 2/9] Add screenshots to UI. Still need to implement resizing. --- lib/roles/device/plugins/screenshot.js | 4 +-- lib/roles/device/support/storage.js | 2 +- lib/roles/storage/temp.js | 21 ++++++------ .../screenshots/screenshots-controller.js | 32 +++++++++++++++++-- .../screenshots/screenshots.jade | 9 +++--- 5 files changed, 49 insertions(+), 19 deletions(-) diff --git a/lib/roles/device/plugins/screenshot.js b/lib/roles/device/plugins/screenshot.js index 1dd2e97a..533db64f 100644 --- a/lib/roles/device/plugins/screenshot.js +++ b/lib/roles/device/plugins/screenshot.js @@ -47,10 +47,10 @@ module.exports = syrup.serial() router.on(wire.ScreenCaptureMessage, function(channel) { var reply = wireutil.reply(options.serial) plugin.capture() - .then(function(url) { + .then(function(file) { push.send([ channel - , reply.okay(url) + , reply.okay('success', file) ]) }) .catch(function(err) { diff --git a/lib/roles/device/support/storage.js b/lib/roles/device/support/storage.js index 7009222b..492ea996 100644 --- a/lib/roles/device/support/storage.js +++ b/lib/roles/device/support/storage.js @@ -32,7 +32,7 @@ module.exports = syrup.serial() else { try { var result = JSON.parse(body) - log.info('Uploaded to %s', result.resources.file) + log.info('Uploaded to %s', result.resources.file.url) resolver.resolve(result.resources.file) } catch (err) { diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index 2454793e..297bf3e4 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -43,15 +43,18 @@ module.exports = function(options) { , resources: (function() { var mapped = Object.create(null) storedFiles.forEach(function(file) { - mapped[file.field] = util.format( - 'http://%s:%s/api/v1/resources/%s%s' - , options.publicIp - , options.port - , file.id - , file.name - ? util.format('/%s', path.basename(file.name)) - : '' - ) + mapped[file.field] = { + date: new Date() + , url: util.format( + 'http://%s:%s/api/v1/resources/%s%s' + , options.publicIp + , options.port + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) + } }) return mapped })() diff --git a/res/app/control-panes/screenshots/screenshots-controller.js b/res/app/control-panes/screenshots/screenshots-controller.js index 71b51eef..18f7aa74 100644 --- a/res/app/control-panes/screenshots/screenshots-controller.js +++ b/res/app/control-panes/screenshots/screenshots-controller.js @@ -1,3 +1,31 @@ -module.exports = function ScreenshotsCtrl($scope) { - +module.exports = function ScreenshotsCtrl($scope, SettingsService) { + $scope.screenshots = [] + $scope.shotSize = 'original' + + $scope.clear = function () { + $scope.screenshots = [] + } + + SettingsService.bind($scope, { + key: 'shotSize' + , storeName: 'ScreenShots.shotSize' + }) + + $scope.shotSizeUrlParameter = function () { + var sizes = { + 'small': '?crop=100x', + 'medium': '?crop=320x', + 'large': '?crop=450x', + 'original': '' + } + return sizes[$scope.shotSize] + } + + $scope.takeScreenShot = function () { + $scope.control.screenshot().then(function(result) { + $scope.$apply(function() { + $scope.screenshots.push(result) + }) + }) + } } diff --git a/res/app/control-panes/screenshots/screenshots.jade b/res/app/control-panes/screenshots/screenshots.jade index aff7c630..74935255 100644 --- a/res/app/control-panes/screenshots/screenshots.jade +++ b/res/app/control-panes/screenshots/screenshots.jade @@ -24,9 +24,8 @@ nothing-to-show(message='{{"No screenshots taken"|translate}}', icon='fa-camera', ng-show='!screenshots.length') ul.screenshots-icon-view.clear-fix li(ng-repeat='shot in screenshots').cursor-select - h4 {{ shot.device.capabilities.info.name.id || shot.device.capabilities.model }} - h5 {{ shot.value.date | date:'yyyy/MM/dd HH:mm:ss' }} - a(ng-href='{{ shot.value.screenshotUrl }}', target='_blank') - img(ng-src='{{ shot.value.screenshotUrl + shotSizeUrlParameter() }}') + h5 {{ shot.body.date | date:'yyyy/MM/dd HH:mm:ss' }} + a(ng-href='{{ shot.body.url }}', target='_blank') + img(ng-src='{{ shot.body.url + shotSizeUrlParameter() }}') - .clearfix \ No newline at end of file + .clearfix From 9e4dc269a21bc30337f6f6a7246be0709a0c437b Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 20 May 2014 11:40:32 +0900 Subject: [PATCH 3/9] Change default screenshot size to "small". --- res/app/control-panes/screenshots/screenshots-controller.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/app/control-panes/screenshots/screenshots-controller.js b/res/app/control-panes/screenshots/screenshots-controller.js index 18f7aa74..89e4de1a 100644 --- a/res/app/control-panes/screenshots/screenshots-controller.js +++ b/res/app/control-panes/screenshots/screenshots-controller.js @@ -1,6 +1,6 @@ module.exports = function ScreenshotsCtrl($scope, SettingsService) { $scope.screenshots = [] - $scope.shotSize = 'original' + $scope.shotSize = 'small' $scope.clear = function () { $scope.screenshots = [] From e56d757cde90bbdb3dd72ea519df9ddb13630d86 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 20 May 2014 19:00:53 +0900 Subject: [PATCH 4/9] Add an app for resizing images. Still needs rate limiting, and still trying to decide how to pass the correct URL to the app. --- lib/cli.js | 26 ++++++++++ lib/roles/storage/plugins/image/index.js | 47 +++++++++++++++++++ lib/roles/storage/plugins/image/param/crop.js | 14 ++++++ .../storage/plugins/image/param/gravity.js | 21 +++++++++ lib/roles/storage/plugins/image/task/get.js | 19 ++++++++ .../storage/plugins/image/task/transform.js | 26 ++++++++++ lib/util/storage.js | 6 ++- 7 files changed, 158 insertions(+), 1 deletion(-) create mode 100644 lib/roles/storage/plugins/image/index.js create mode 100644 lib/roles/storage/plugins/image/param/crop.js create mode 100644 lib/roles/storage/plugins/image/param/gravity.js create mode 100644 lib/roles/storage/plugins/image/task/get.js create mode 100644 lib/roles/storage/plugins/image/task/transform.js diff --git a/lib/cli.js b/lib/cli.js index 32e92ea0..7488af5e 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -500,6 +500,32 @@ program }) }) +program + .command('storage-plugin-image') + .description('start storage image plugin') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) + .option('--cache-dir ' + , 'where to cache images' + , String + , os.tmpdir()) + .action(function(options) { + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } + + require('./roles/storage/plugins/image')({ + port: options.port + , storageUrl: options.storageUrl + , cacheDir: options.cacheDir + }) + }) + program .command('migrate') .description('migrates the database to the latest version') diff --git a/lib/roles/storage/plugins/image/index.js b/lib/roles/storage/plugins/image/index.js new file mode 100644 index 00000000..8f7304e6 --- /dev/null +++ b/lib/roles/storage/plugins/image/index.js @@ -0,0 +1,47 @@ +var http = require('http') +var util = require('util') + +var express = require('express') +var Promise = require('bluebird') +var gm = require('gm') + +var logger = require('../../../../util/logger') + +var parseCrop = require('./param/crop') +var parseGravity = require('./param/gravity') +var get = require('./task/get') +var transform = require('./task/transform') + +module.exports = function(options) { + var log = logger.createLogger('storage:plugins:image') + , app = express() + , server = http.createServer(app) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.get('/api/v1/resources/:id/*', function(req, res) { + get(req.params.id, options) + .then(function(stream) { + return transform(stream, { + crop: parseCrop(req.query.crop) + , gravity: parseGravity(req.query.gravity) + }) + }) + .then(function(out) { + res.status(200) + out.pipe(res) + }) + .catch(function(err) { + log.error('Unable to transform resource "%s"', req.params.id, err.stack) + res.status(500) + .json({ + success: false + }) + }) + }) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/storage/plugins/image/param/crop.js b/lib/roles/storage/plugins/image/param/crop.js new file mode 100644 index 00000000..f719850f --- /dev/null +++ b/lib/roles/storage/plugins/image/param/crop.js @@ -0,0 +1,14 @@ +var RE_CROP = /^([0-9]*)x([0-9]*)$/ + +module.exports = function(raw) { + var parsed + + if (raw && (parsed = RE_CROP.exec(raw))) { + return { + width: +parsed[1] || 0 + , height: +parsed[2] || 0 + } + } + + return null +} diff --git a/lib/roles/storage/plugins/image/param/gravity.js b/lib/roles/storage/plugins/image/param/gravity.js new file mode 100644 index 00000000..bb6ecb6f --- /dev/null +++ b/lib/roles/storage/plugins/image/param/gravity.js @@ -0,0 +1,21 @@ +var GRAVITY = { + northwest: 'NorthWest' +, north: 'North' +, northeast: 'NorthEast' +, west: 'West' +, center: 'Center' +, east: 'East' +, southwest: 'SouthWest' +, south: 'South' +, southeast: 'SouthEast' +} + +module.exports = function(raw) { + var parsed + + if (raw && (parsed = GRAVITY[raw])) { + return parsed + } + + return null +} diff --git a/lib/roles/storage/plugins/image/task/get.js b/lib/roles/storage/plugins/image/task/get.js new file mode 100644 index 00000000..32214dec --- /dev/null +++ b/lib/roles/storage/plugins/image/task/get.js @@ -0,0 +1,19 @@ +var util = require('util') +var http = require('http') + +var Promise = require('bluebird') + +module.exports = function(id, options) { + return new Promise(function(resolve, reject) { + http.get(util.format('%sapi/v1/resources/%s', options.storageUrl, id)) + .on('response', function(res) { + if (res.statusCode !== 200) { + reject(new Error(util.format('HTTP %d', res.statusCode))) + } + else { + resolve(res) + } + }) + .on('error', reject) + }) +} diff --git a/lib/roles/storage/plugins/image/task/transform.js b/lib/roles/storage/plugins/image/task/transform.js new file mode 100644 index 00000000..b87ccbbe --- /dev/null +++ b/lib/roles/storage/plugins/image/task/transform.js @@ -0,0 +1,26 @@ +var gm = require('gm') +var Promise = require('bluebird') + +module.exports = function(stream, options) { + return new Promise(function(resolve, reject) { + var transform = gm(stream) + + if (options.gravity) { + transform.gravity(options.gravity) + } + + if (options.crop) { + transform.geometry(options.crop.width, options.crop.height, '^') + transform.crop(options.crop.width, options.crop.height, 0, 0) + } + + transform.stream(function(err, stdout) { + if (err) { + reject(err) + } + else { + resolve(stdout) + } + }) + }) +} diff --git a/lib/util/storage.js b/lib/util/storage.js index 3ea23ebf..ec8fd7d1 100644 --- a/lib/util/storage.js +++ b/lib/util/storage.js @@ -14,14 +14,18 @@ util.inherits(Storage, events.EventEmitter) Storage.prototype.store = function(file) { var id = uuid.v4() + this.set(id, file) + return id +} +Storage.prototype.set = function(id, file) { this.files[id] = { timeout: 600000 , lastActivity: Date.now() , data: file } - return id + return file } Storage.prototype.remove = function(id) { From c0d02c4e3a82ce4f132900e7322b1c7ff19b9d3b Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Wed, 21 May 2014 14:07:53 +0900 Subject: [PATCH 5/9] Screenshot resizing works in the UI now. Still missing rate limiting. --- lib/cli.js | 26 +++++++++++--- lib/roles/app.js | 6 ++-- lib/roles/device/plugins/screenshot.js | 2 +- lib/roles/device/support/storage.js | 6 ++-- lib/roles/storage/plugins/image/index.js | 7 ++-- lib/roles/storage/plugins/image/task/get.js | 5 +-- lib/roles/storage/temp.js | 36 +++++-------------- .../screenshots/screenshots.jade | 2 +- 8 files changed, 43 insertions(+), 47 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 7488af5e..f63f9459 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -400,6 +400,9 @@ program .option('-r, --storage-url ' , 'URL to storage client' , String) + .option('--storage-plugin-image-url ' + , 'URL to image storage plugin' + , String) .option('-u, --connect-sub ' , 'sub endpoint' , cliutil.list) @@ -418,6 +421,9 @@ program if (!options.storageUrl) { this.missingArgument('--storage-url') } + if (!options.storagePluginImageUrl) { + this.missingArgument('--storage-plugin-image-url') + } if (!options.connectSub) { this.missingArgument('--connect-sub') } @@ -431,6 +437,7 @@ program , ssid: options.ssid , authUrl: options.authUrl , storageUrl: options.storageUrl + , storagePluginImageUrl: options.storagePluginImageUrl , endpoints: { sub: options.connectSub , push: options.connectPush @@ -484,10 +491,6 @@ program , 'port (or $PORT)' , Number , process.env.PORT || 7100) - .option('--public-ip ' - , 'public ip for global access' - , String - , ip()) .option('--save-dir ' , 'where to save files' , String @@ -495,7 +498,6 @@ program .action(function(options) { require('./roles/storage/temp')({ port: options.port - , publicIp: options.publicIp , saveDir: options.saveDir }) }) @@ -586,6 +588,10 @@ program , 'storage port' , Number , 7102) + .option('--storage-plugin-image-port ' + , 'storage image plugin port' + , Number + , 7103) .option('--provider ' , 'provider name (or os.hostname())' , String @@ -676,6 +682,8 @@ program , '--auth-url', util.format('http://localhost:%d/', options.authPort) , '--storage-url' , util.format('http://localhost:%d/', options.storagePort) + , '--storage-plugin-image-url' + , util.format('http://localhost:%d/', options.storagePluginImagePort) , '--connect-sub', options.bindAppPub , '--connect-push', options.bindAppPull ].concat((function() { @@ -691,6 +699,14 @@ program 'storage-temp' , '--port', options.storagePort ]) + + // image processor + , procutil.fork(__filename, [ + 'storage-plugin-image' + , '--port', options.storagePluginImagePort + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) + ]) ] function shutdown() { diff --git a/lib/roles/app.js b/lib/roles/app.js index c8642d5a..71c2b1f7 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -74,13 +74,13 @@ module.exports = function(options) { })) // Proxied requests must come before any body parsers - app.post('/api/v1/resources', function(req, res) { + app.all('/api/v1/s/image/*', function(req, res) { proxy.web(req, res, { - target: options.storageUrl + target: options.storagePluginImageUrl }) }) - app.get('/api/v1/resources/:id', function(req, res) { + app.all('/api/v1/s/*', function(req, res) { proxy.web(req, res, { target: options.storageUrl }) diff --git a/lib/roles/device/plugins/screenshot.js b/lib/roles/device/plugins/screenshot.js index 533db64f..4d23bdbe 100644 --- a/lib/roles/device/plugins/screenshot.js +++ b/lib/roles/device/plugins/screenshot.js @@ -31,7 +31,7 @@ module.exports = syrup.serial() ))) } else { - resolve(storage.store(res, { + resolve(storage.store('image', res, { filename: util.format('%s.png', options.serial) , contentType: 'image/png' , knownLength: +res.headers['content-length'] diff --git a/lib/roles/device/support/storage.js b/lib/roles/device/support/storage.js index 492ea996..6815c80c 100644 --- a/lib/roles/device/support/storage.js +++ b/lib/roles/device/support/storage.js @@ -11,11 +11,11 @@ module.exports = syrup.serial() var log = logger.createLogger('device:support:storage') var plugin = Object.create(null) - plugin.store = function(stream, meta) { + plugin.store = function(type, stream, meta) { var resolver = Promise.defer() var req = request.post({ - url: util.format('%sapi/v1/resources', options.storageUrl) + url: util.format('%sapi/v1/s/%s', options.storageUrl, type) } , function(err, res, body) { if (err) { @@ -32,7 +32,7 @@ module.exports = syrup.serial() else { try { var result = JSON.parse(body) - log.info('Uploaded to %s', result.resources.file.url) + log.info('Uploaded to %s', result.resources.file.href) resolver.resolve(result.resources.file) } catch (err) { diff --git a/lib/roles/storage/plugins/image/index.js b/lib/roles/storage/plugins/image/index.js index 8f7304e6..40a30d34 100644 --- a/lib/roles/storage/plugins/image/index.js +++ b/lib/roles/storage/plugins/image/index.js @@ -1,9 +1,6 @@ var http = require('http') -var util = require('util') var express = require('express') -var Promise = require('bluebird') -var gm = require('gm') var logger = require('../../../../util/logger') @@ -21,8 +18,8 @@ module.exports = function(options) { app.set('case sensitive routing', true) app.set('trust proxy', true) - app.get('/api/v1/resources/:id/*', function(req, res) { - get(req.params.id, options) + app.get('/api/v1/s/image/:id/*', function(req, res) { + get(req.url, options) .then(function(stream) { return transform(stream, { crop: parseCrop(req.query.crop) diff --git a/lib/roles/storage/plugins/image/task/get.js b/lib/roles/storage/plugins/image/task/get.js index 32214dec..73a12ea2 100644 --- a/lib/roles/storage/plugins/image/task/get.js +++ b/lib/roles/storage/plugins/image/task/get.js @@ -1,11 +1,12 @@ var util = require('util') var http = require('http') +var url = require('url') var Promise = require('bluebird') -module.exports = function(id, options) { +module.exports = function(path, options) { return new Promise(function(resolve, reject) { - http.get(util.format('%sapi/v1/resources/%s', options.storageUrl, id)) + http.get(url.resolve(options.storageUrl, path)) .on('response', function(res) { if (res.statusCode !== 200) { reject(new Error(util.format('HTTP %d', res.statusCode))) diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index 297bf3e4..a4a28379 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -7,11 +7,10 @@ var formidable = require('formidable') var Promise = require('bluebird') var logger = require('../../util/logger') -var requtil = require('../../util/requtil') var Storage = require('../../util/storage') module.exports = function(options) { - var log = logger.createLogger('storage-temp') + var log = logger.createLogger('storage:temp') , app = express() , server = http.createServer(app) , storage = new Storage() @@ -24,7 +23,7 @@ module.exports = function(options) { log.info('Cleaning up inactive resource "%s"', id) }) - app.post('/api/v1/resources', function(req, res) { + app.post('/api/v1/s/:type', function(req, res) { var form = new formidable.IncomingForm() Promise.promisify(form.parse, form)(req) .spread(function(fields, files) { @@ -45,10 +44,12 @@ module.exports = function(options) { storedFiles.forEach(function(file) { mapped[file.field] = { date: new Date() - , url: util.format( - 'http://%s:%s/api/v1/resources/%s%s' - , options.publicIp - , options.port + , type: req.params.type + , id: file.id + , name: file.name + , href: util.format( + '/api/v1/s/%s/%s%s' + , req.params.type , file.id , file.name ? util.format('/%s', path.basename(file.name)) @@ -60,14 +61,6 @@ module.exports = function(options) { })() }) }) - .catch(requtil.ValidationError, function(err) { - res.status(400) - .json({ - success: false - , error: 'ValidationError' - , validationErrors: err.errors - }) - }) .catch(function(err) { log.error('Error storing resource', err.stack) res.status(500) @@ -78,18 +71,7 @@ module.exports = function(options) { }) }) - app.get('/api/v1/resources/:id', function(req, res) { - var file = storage.retrieve(req.params.id) - if (file) { - res.set('Content-Type', file.type) - res.sendfile(file.path) - } - else { - res.send(404) - } - }) - - app.get('/api/v1/resources/:id/*', function(req, res) { + app.get('/api/v1/s/:type/:id/*', function(req, res) { var file = storage.retrieve(req.params.id) if (file) { res.set('Content-Type', file.type) diff --git a/res/app/control-panes/screenshots/screenshots.jade b/res/app/control-panes/screenshots/screenshots.jade index 74935255..053f5680 100644 --- a/res/app/control-panes/screenshots/screenshots.jade +++ b/res/app/control-panes/screenshots/screenshots.jade @@ -26,6 +26,6 @@ li(ng-repeat='shot in screenshots').cursor-select h5 {{ shot.body.date | date:'yyyy/MM/dd HH:mm:ss' }} a(ng-href='{{ shot.body.url }}', target='_blank') - img(ng-src='{{ shot.body.url + shotSizeUrlParameter() }}') + img(ng-src='{{ shot.body.href + shotSizeUrlParameter() }}') .clearfix From 98470cda20837b2a2f9c9b4cb0852b313c4e2f66 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Wed, 21 May 2014 14:32:21 +0900 Subject: [PATCH 6/9] Add rate limiting to the image processor. --- lib/cli.js | 4 +++ lib/roles/storage/plugins/image/index.js | 46 ++++++++++++++---------- lib/util/requtil.js | 22 ++++++++++++ 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index f63f9459..2792e582 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -512,6 +512,9 @@ program .option('-r, --storage-url ' , 'URL to storage client' , String) + .option('-c, --concurrency ' + , 'maximum number of simultaneous transformations' + , Number) .option('--cache-dir ' , 'where to cache images' , String @@ -525,6 +528,7 @@ program port: options.port , storageUrl: options.storageUrl , cacheDir: options.cacheDir + , concurrency: options.concurrency || os.cpus().length }) }) diff --git a/lib/roles/storage/plugins/image/index.js b/lib/roles/storage/plugins/image/index.js index 40a30d34..9fe4d2b9 100644 --- a/lib/roles/storage/plugins/image/index.js +++ b/lib/roles/storage/plugins/image/index.js @@ -3,6 +3,7 @@ var http = require('http') var express = require('express') var logger = require('../../../../util/logger') +var requtil = require('../../../../util/requtil') var parseCrop = require('./param/crop') var parseGravity = require('./param/gravity') @@ -18,26 +19,33 @@ module.exports = function(options) { app.set('case sensitive routing', true) app.set('trust proxy', true) - app.get('/api/v1/s/image/:id/*', function(req, res) { - get(req.url, options) - .then(function(stream) { - return transform(stream, { - crop: parseCrop(req.query.crop) - , gravity: parseGravity(req.query.gravity) - }) - }) - .then(function(out) { - res.status(200) - out.pipe(res) - }) - .catch(function(err) { - log.error('Unable to transform resource "%s"', req.params.id, err.stack) - res.status(500) - .json({ - success: false + app.get( + '/api/v1/s/image/:id/*' + , requtil.limit(options.concurrency, function(req, res) { + return get(req.url, options) + .then(function(stream) { + return transform(stream, { + crop: parseCrop(req.query.crop) + , gravity: parseGravity(req.query.gravity) }) - }) - }) + }) + .then(function(out) { + res.status(200) + out.pipe(res) + }) + .catch(function(err) { + log.error( + 'Unable to transform resource "%s"' + , req.params.id + , err.stack + ) + res.status(500) + .json({ + success: false + }) + }) + }) + ) server.listen(options.port) log.info('Listening on port %d', options.port) diff --git a/lib/util/requtil.js b/lib/util/requtil.js index 35390962..79d85210 100644 --- a/lib/util/requtil.js +++ b/lib/util/requtil.js @@ -26,3 +26,25 @@ module.exports.validate = function(req, rules) { } }) } + +module.exports.limit = function(limit, handler) { + var queue = [] + var running = 0 + + function done() { + running -= 1 + maybeNext() + } + + function maybeNext() { + while (running < limit && queue.length) { + running += 1 + handler.apply(null, queue.shift()).finally(done) + } + } + + return function() { + queue.push(arguments) + maybeNext() + } +} From 7d36f27a315e619b63a47834ddf3f414442b6466 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Wed, 21 May 2014 18:07:24 +0900 Subject: [PATCH 7/9] Fix screenshot link href. --- res/app/control-panes/screenshots/screenshots.jade | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/res/app/control-panes/screenshots/screenshots.jade b/res/app/control-panes/screenshots/screenshots.jade index 053f5680..832eefcb 100644 --- a/res/app/control-panes/screenshots/screenshots.jade +++ b/res/app/control-panes/screenshots/screenshots.jade @@ -25,7 +25,7 @@ ul.screenshots-icon-view.clear-fix li(ng-repeat='shot in screenshots').cursor-select h5 {{ shot.body.date | date:'yyyy/MM/dd HH:mm:ss' }} - a(ng-href='{{ shot.body.url }}', target='_blank') + a(ng-href='{{ shot.body.href }}', target='_blank') img(ng-src='{{ shot.body.href + shotSizeUrlParameter() }}') .clearfix From 1db48e9fcb6637728208bfa5c680d7a0b7084764 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Wed, 21 May 2014 20:12:34 +0900 Subject: [PATCH 8/9] Forgot we are getting screenshots as JPG. --- lib/roles/device/plugins/screenshot.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/roles/device/plugins/screenshot.js b/lib/roles/device/plugins/screenshot.js index 4d23bdbe..2b32ac10 100644 --- a/lib/roles/device/plugins/screenshot.js +++ b/lib/roles/device/plugins/screenshot.js @@ -32,8 +32,8 @@ module.exports = syrup.serial() } else { resolve(storage.store('image', res, { - filename: util.format('%s.png', options.serial) - , contentType: 'image/png' + filename: util.format('%s.jpg', options.serial) + , contentType: 'image/jpeg' , knownLength: +res.headers['content-length'] })) } From 41ed33f5c47595b96a8c8a37211a1445020befdf Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Thu, 22 May 2014 13:33:38 +0900 Subject: [PATCH 9/9] Implement APK uploads using the new storage system. Installation from URL still does not work, and dropping the file on the screen may not work either. --- bower.json | 2 +- lib/cli.js | 85 ++++--- lib/roles/app.js | 16 +- lib/roles/cache/apk.js | 216 ------------------ lib/roles/device/plugins/install.js | 40 +++- lib/roles/storage/plugins/apk/index.js | 54 +++++ .../storage/plugins/apk/task/manifest.js | 19 ++ lib/roles/storage/temp.js | 65 +++++- lib/util/download.js | 67 ++++++ lib/wire/wire.proto | 5 +- res/app/app.js | 3 +- .../components/stf/control/control-service.js | 43 +--- res/app/components/stf/control/index.js | 1 + .../components/stf/control/storage-service.js | 40 ++++ .../dashboard/upload/upload-controller.js | 62 +++-- .../dashboard/upload/upload.jade | 4 +- webpack.config.js | 3 +- 17 files changed, 389 insertions(+), 336 deletions(-) delete mode 100644 lib/roles/cache/apk.js create mode 100644 lib/roles/storage/plugins/apk/index.js create mode 100644 lib/roles/storage/plugins/apk/task/manifest.js create mode 100644 lib/util/download.js create mode 100644 res/app/components/stf/control/storage-service.js diff --git a/bower.json b/bower.json index 439d65fa..e39bd8fb 100644 --- a/bower.json +++ b/bower.json @@ -24,7 +24,7 @@ "stf-graphics": "git@ghe.amb.ca.local:stf/stf-graphics.git", "angular-bootstrap": "~0.11.0", "angular-dialog-service": "~5.0.0", - "ng-file-upload": "~1.2.11", + "ng-file-upload": "~1.4.0", "angular-growl-v2": "JanStevens/angular-growl-2#~0.6.0", "bluebird": "~1.2.4", "angular-tree-control": "~0.1.5", diff --git a/lib/cli.js b/lib/cli.js index 2792e582..169e147f 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -403,6 +403,9 @@ program .option('--storage-plugin-image-url ' , 'URL to image storage plugin' , String) + .option('--storage-plugin-apk-url ' + , 'URL to apk storage plugin' + , String) .option('-u, --connect-sub ' , 'sub endpoint' , cliutil.list) @@ -424,6 +427,9 @@ program if (!options.storagePluginImageUrl) { this.missingArgument('--storage-plugin-image-url') } + if (!options.storagePluginApkUrl) { + this.missingArgument('--storage-plugin-apk-url') + } if (!options.connectSub) { this.missingArgument('--connect-sub') } @@ -438,6 +444,7 @@ program , authUrl: options.authUrl , storageUrl: options.storageUrl , storagePluginImageUrl: options.storagePluginImageUrl + , storagePluginApkUrl: options.storagePluginApkUrl , endpoints: { sub: options.connectSub , push: options.connectPush @@ -446,44 +453,6 @@ program }) }) -program - .command('cache-apk') - .description('apk cache') - .option('-p, --port ' - , 'port (or $PORT)' - , Number - , process.env.PORT || 7100) - .option('--public-ip ' - , 'public ip for global access' - , String - , ip()) - .option('--save-dir ' - , 'where to save files' - , String - , os.tmpdir()) - .option('--id ' - , 'communication identifier' - , String - , 'storage') - .option('--connect-push ' - , 'push endpoint' - , cliutil.list) - .action(function(options) { - if (!options.connectPush) { - this.missingArgument('--connect-push') - } - - require('./roles/storage/temp')({ - port: options.port - , publicIp: options.publicIp - , saveDir: options.saveDir - , id: options.id - , endpoints: { - push: options.connectPush - } - }) - }) - program .command('storage-temp') .description('start temp storage') @@ -532,6 +501,32 @@ program }) }) +program + .command('storage-plugin-apk') + .description('start storage apk plugin') + .option('-p, --port ' + , 'port (or $PORT)' + , Number + , process.env.PORT || 7100) + .option('-r, --storage-url ' + , 'URL to storage client' + , String) + .option('--cache-dir ' + , 'where to cache images' + , String + , os.tmpdir()) + .action(function(options) { + if (!options.storageUrl) { + this.missingArgument('--storage-url') + } + + require('./roles/storage/plugins/apk')({ + port: options.port + , storageUrl: options.storageUrl + , cacheDir: options.cacheDir + }) + }) + program .command('migrate') .description('migrates the database to the latest version') @@ -596,6 +591,10 @@ program , 'storage image plugin port' , Number , 7103) + .option('--storage-plugin-apk-port ' + , 'storage apk plugin port' + , Number + , 7104) .option('--provider ' , 'provider name (or os.hostname())' , String @@ -688,6 +687,8 @@ program , util.format('http://localhost:%d/', options.storagePort) , '--storage-plugin-image-url' , util.format('http://localhost:%d/', options.storagePluginImagePort) + , '--storage-plugin-apk-url' + , util.format('http://localhost:%d/', options.storagePluginApkPort) , '--connect-sub', options.bindAppPub , '--connect-push', options.bindAppPull ].concat((function() { @@ -711,6 +712,14 @@ program , '--storage-url' , util.format('http://localhost:%d/', options.storagePort) ]) + + // apk processor + , procutil.fork(__filename, [ + 'storage-plugin-apk' + , '--port', options.storagePluginApkPort + , '--storage-url' + , util.format('http://localhost:%d/', options.storagePort) + ]) ] function shutdown() { diff --git a/lib/roles/app.js b/lib/roles/app.js index 71c2b1f7..3ec0f499 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -73,13 +73,21 @@ module.exports = function(options) { , authUrl: options.authUrl })) - // Proxied requests must come before any body parsers + // Proxied requests must come before any body parsers. These proxies are + // here mainly for convenience, they should be replaced with proper reverse + // proxies in production. app.all('/api/v1/s/image/*', function(req, res) { proxy.web(req, res, { target: options.storagePluginImageUrl }) }) + app.all('/api/v1/s/apk/*', function(req, res) { + proxy.web(req, res, { + target: options.storagePluginApkUrl + }) + }) + app.all('/api/v1/s/*', function(req, res) { proxy.web(req, res, { target: options.storageUrl @@ -507,7 +515,11 @@ module.exports = function(options) { channel , wireutil.transaction( responseChannel - , new wire.InstallMessage(data) + , new wire.InstallMessage( + data.href + , data.launch === true + , JSON.stringify(data.manifest) + ) ) ]) }) diff --git a/lib/roles/cache/apk.js b/lib/roles/cache/apk.js deleted file mode 100644 index 718dcb15..00000000 --- a/lib/roles/cache/apk.js +++ /dev/null @@ -1,216 +0,0 @@ -var http = require('http') -var util = require('util') -var fs = require('fs') - -var express = require('express') -var validator = require('express-validator') -var Promise = require('bluebird') -var ApkReader = require('adbkit-apkreader') -var request = require('request') -var progress = require('request-progress') -var temp = require('temp') -var zmq = require('zmq') - -var logger = require('../../util/logger') -var requtil = require('../../util/requtil') -var Storage = require('../../util/storage') -var wireutil = require('../../wire/util') - -module.exports = function(options) { - var log = logger.createLogger('cache-apk') - , app = express() - , server = http.createServer(app) - , storage = new Storage() - - // Output - var push = zmq.socket('push') - options.endpoints.push.forEach(function(endpoint) { - log.info('Sending output to %s', endpoint) - push.connect(endpoint) - }) - - app.set('strict routing', true) - app.set('case sensitive routing', true) - app.set('trust proxy', true) - - app.use(express.json()) - app.use(validator()) - - storage.on('timeout', function(id) { - log.info('Cleaning up inactive resource "%s"', id) - }) - - function processFile(file) { - var resolver = Promise.defer() - - log.info('Processing file "%s"', file.path) - - resolver.progress({ - percent: 0 - }) - - process.nextTick(function() { - try { - var reader = ApkReader.readFile(file.path) - var manifest = reader.readManifestSync() - resolver.resolve(manifest) - } - catch (err) { - err.reportCode = 'fail_invalid_app_file' - resolver.reject(err) - } - }) - - return resolver.promise - } - - function storeFile(file) { - var id = storage.store(file) - return Promise.resolve({ - id: id - , url: util.format( - 'http://%s:%s/api/v1/resources/%s' - , options.publicIp - , options.port - , id - ) - }) - } - - function download(url) { - var resolver = Promise.defer() - var path = temp.path({ - dir: options.saveDir - }) - - log.info('Downloading "%s" to "%s"', url, path) - - function errorListener(err) { - err.reportCode = 'fail_download' - resolver.reject(err) - } - - function progressListener(state) { - resolver.progress(state) - } - - function closeListener() { - resolver.resolve({ - path: path - }) - } - - resolver.progress({ - percent: 0 - }) - - try { - var req = progress(request(url), { - throttle: 100 // Throttle events, not upload speed - }) - .on('progress', progressListener) - - var save = req.pipe(fs.createWriteStream(path)) - .on('error', errorListener) - .on('close', closeListener) - } - catch (err) { - err.reportCode = 'fail_invalid_url' - resolver.reject(err) - } - - return resolver.promise.finally(function() { - req.removeListener('progress', progressListener) - save.removeListener('error', errorListener) - save.removeListener('close', closeListener) - }) - } - - app.post('/api/v1/cache', function(req, res) { - var reply = wireutil.reply(options.id) - - function sendProgress(data, progress) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.progress(data, progress) - ]) - } - } - - function sendDone(success, data, body) { - if (req.query.channel) { - push.send([ - req.query.channel - , reply.okay(data, body) - ]) - } - } - - requtil.validate(req, function() { - req.checkQuery('channel').notEmpty() - }) - .then(function() { - return requtil.validate(req, function() { - req.checkBody('url').notEmpty() - }) - .then(function() { - return download(req.body.url) - .progressed(function(progress) { - sendProgress('uploading', 0.7 * progress.percent) - }) - }) - }) - .then(function(file) { - return processFile(file) - .progressed(function(progress) { - sendProgress('processing', 70 + 0.2 * progress.percent) - }) - .then(function(manifest) { - sendProgress('storing', 90) - return storeFile(file) - .then(function(data) { - data.manifest = manifest - return data - }) - }) - }) - .then(function(data) { - sendDone(true, 'success', data) - data.success = true - res.json(201, data) - }) - .catch(requtil.ValidationError, function(err) { - sendDone(false, err.reportCode || 'fail_validation') - res.status(400) - .json({ - success: false - , error: 'ValidationError' - , validationErrors: err.errors - }) - }) - .catch(function(err) { - log.error('Unexpected error', err.stack) - sendDone(false, err.reportCode || 'fail') - res.status(500) - .json({ - success: false - , error: 'ServerError' - }) - }) - }) - - app.get('/api/v1/cache/:id', function(req, res) { - var file = storage.retrieve(req.params.id) - if (file) { - res.set('Content-Type', file.type) - res.sendfile(file.path) - } - else { - res.send(404) - } - }) - - server.listen(options.port) - log.info('Listening on port %d', options.port) -} diff --git a/lib/roles/device/plugins/install.js b/lib/roles/device/plugins/install.js index 73283116..71d8b01b 100644 --- a/lib/roles/device/plugins/install.js +++ b/lib/roles/device/plugins/install.js @@ -1,4 +1,6 @@ var stream = require('stream') +var url = require('url') +var util = require('util') var syrup = require('syrup') var request = require('request') @@ -17,7 +19,7 @@ module.exports = syrup.serial() var log = logger.createLogger('device:plugins:install') router.on(wire.InstallMessage, function(channel, message) { - log.info('Installing "%s"', message.url) + log.info('Installing "%s"', message.href) var reply = wireutil.reply(options.serial) @@ -30,7 +32,7 @@ module.exports = syrup.serial() function pushApp() { var req = request({ - url: message.url + url: url.resolve(options.storageUrl, message.href) }) // We need to catch the Content-Length on the fly or we risk @@ -104,16 +106,30 @@ module.exports = syrup.serial() .timeout(30000) }) .then(function() { - if (message.launchActivity) { - log.info( - 'Launching activity with action "%s" on component "%s"' - , message.launchActivity.action - , message.launchActivity.component - ) - // Progress 90% - sendProgress('launching_app', 90) - return adb.startActivity(options.serial, message.launchActivity) - .timeout(15000) + if (message.launch) { + var manifest = JSON.parse(message.manifest) + if (manifest.application.launcherActivities.length) { + var launchActivity = { + action: 'android.intent.action.MAIN' + , component: util.format( + '%s/%s' + , manifest.package + , manifest.application.launcherActivities[0].name + ) + , category: ['android.intent.category.LAUNCHER'] + , flags: 0x10200000 + } + + log.info( + 'Launching activity with action "%s" on component "%s"' + , launchActivity.action + , launchActivity.component + ) + // Progress 90% + sendProgress('launching_app', 90) + return adb.startActivity(options.serial, launchActivity) + .timeout(15000) + } } }) .then(function() { diff --git a/lib/roles/storage/plugins/apk/index.js b/lib/roles/storage/plugins/apk/index.js new file mode 100644 index 00000000..6f34c731 --- /dev/null +++ b/lib/roles/storage/plugins/apk/index.js @@ -0,0 +1,54 @@ +var http = require('http') +var url = require('url') + +var express = require('express') +var httpProxy = require('http-proxy') + +var logger = require('../../../../util/logger') +var download = require('../../../../util/download') +var manifest = require('./task/manifest') + +module.exports = function(options) { + var log = logger.createLogger('storage:plugins:apk') + , app = express() + , server = http.createServer(app) + , proxy = httpProxy.createProxyServer() + + proxy.on('error', function(err) { + log.error('Proxy had an error', err.stack) + }) + + app.set('strict routing', true) + app.set('case sensitive routing', true) + app.set('trust proxy', true) + + app.get('/api/v1/s/apk/:id/*/manifest', function(req, res) { + download(url.resolve(options.storageUrl, req.url), { + dir: options.cacheDir + }) + .then(manifest) + .then(function(data) { + res.status(200) + .json({ + success: true + , manifest: data + }) + }) + .catch(function(err) { + log.error('Unable to read manifest of "%s"', req.params.id, err.stack) + res.status(500) + .json({ + success: false + }) + }) + }) + + app.get('/api/v1/s/apk/:id/*', function(req, res) { + proxy.web(req, res, { + target: options.storageUrl + }) + }) + + server.listen(options.port) + log.info('Listening on port %d', options.port) +} diff --git a/lib/roles/storage/plugins/apk/task/manifest.js b/lib/roles/storage/plugins/apk/task/manifest.js new file mode 100644 index 00000000..6a8f8efb --- /dev/null +++ b/lib/roles/storage/plugins/apk/task/manifest.js @@ -0,0 +1,19 @@ +var Promise = require('bluebird') +var ApkReader = require('adbkit-apkreader') + +module.exports = function(file) { + var resolver = Promise.defer() + + process.nextTick(function() { + try { + var reader = ApkReader.readFile(file.path) + var manifest = reader.readManifestSync() + resolver.resolve(manifest) + } + catch (err) { + resolver.reject(err) + } + }) + + return resolver.promise +} diff --git a/lib/roles/storage/temp.js b/lib/roles/storage/temp.js index a4a28379..3681cc06 100644 --- a/lib/roles/storage/temp.js +++ b/lib/roles/storage/temp.js @@ -3,11 +3,14 @@ var util = require('util') var path = require('path') var express = require('express') +var validator = require('express-validator') var formidable = require('formidable') var Promise = require('bluebird') var logger = require('../../util/logger') var Storage = require('../../util/storage') +var requtil = require('../../util/requtil') +var download = require('../../util/download') module.exports = function(options) { var log = logger.createLogger('storage:temp') @@ -19,19 +22,77 @@ module.exports = function(options) { app.set('case sensitive routing', true) app.set('trust proxy', true) + app.use(express.json()) + app.use(validator()) + storage.on('timeout', function(id) { log.info('Cleaning up inactive resource "%s"', id) }) + app.post('/api/v1/s/:type/download', function(req, res) { + requtil.validate(req, function() { + req.checkBody('url').notEmpty() + }) + .then(function() { + return download(req.body.url, { + dir: options.cacheDir + }) + }) + .then(function(file) { + return { + id: storage.store(file) + , name: file.name + } + }) + .then(function(file) { + res.status(201) + .json({ + success: true + , resource: { + date: new Date() + , type: req.params.type + , id: file.id + , name: file.name + , href: util.format( + '/api/v1/s/%s/%s%s' + , req.params.type + , file.id + , file.name + ? util.format('/%s', path.basename(file.name)) + : '' + ) + } + }) + }) + .catch(requtil.ValidationError, function(err) { + res.status(400) + .json({ + success: false + , error: 'ValidationError' + , validationErrors: err.errors + }) + }) + .catch(function(err) { + log.error('Error storing resource', err.stack) + res.status(500) + .json({ + success: false + , error: 'ServerError' + }) + }) + }) + app.post('/api/v1/s/:type', function(req, res) { var form = new formidable.IncomingForm() Promise.promisify(form.parse, form)(req) .spread(function(fields, files) { return Object.keys(files).map(function(field) { + var file = files[field] + log.info('Uploaded "%s" to "%s"', file.name, file.path) return { field: field - , id: storage.store(files[field]) - , name: files[field].name + , id: storage.store(file) + , name: file.name } }) }) diff --git a/lib/util/download.js b/lib/util/download.js new file mode 100644 index 00000000..372345e8 --- /dev/null +++ b/lib/util/download.js @@ -0,0 +1,67 @@ +var fs = require('fs') + +var Promise = require('bluebird') +var request = require('request') +var progress = require('request-progress') +var temp = require('temp') + +module.exports = function download(url, options) { + var resolver = Promise.defer() + var path = temp.path(options) + + function errorListener(err) { + resolver.reject(err) + } + + function progressListener(state) { + if (state.total !== null) { + resolver.progress({ + lengthComputable: true + , loaded: state.received + , total: state.total + }) + } + else { + resolver.progress({ + lengthComputable: false + , loaded: state.received + , total: state.received + }) + } + } + + function closeListener() { + resolver.resolve({ + path: path + }) + } + + resolver.progress({ + percent: 0 + }) + + try { + var req = progress(request(url), { + throttle: 100 // Throttle events, not upload speed + }) + .on('progress', progressListener) + + resolver.promise.finally(function() { + req.removeListener('progress', progressListener) + }) + + var save = req.pipe(fs.createWriteStream(path)) + .on('error', errorListener) + .on('close', closeListener) + + resolver.promise.finally(function() { + save.removeListener('error', errorListener) + save.removeListener('close', closeListener) + }) + } + catch (err) { + resolver.reject(err) + } + + return resolver.promise +} diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index f8e67fea..eb5971e2 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -354,8 +354,9 @@ message ShellKeepAliveMessage { } message InstallMessage { - required string url = 1; - optional LaunchActivityMessage launchActivity = 2; + required string href = 1; + required bool launch = 2; + optional string manifest = 3; } message UninstallMessage { diff --git a/res/app/app.js b/res/app/app.js index c36e03e0..ec9d0a66 100644 --- a/res/app/app.js +++ b/res/app/app.js @@ -3,7 +3,8 @@ require('angular-route') require('angular-touch') require('angular-gettext') -require('ng-file-upload') +require('ng-file-upload-shim5') +require('ng-file-upload-main') angular.module('app', [ 'ngRoute', diff --git a/res/app/components/stf/control/control-service.js b/res/app/components/stf/control/control-service.js index 40f87f3f..5f9661e9 100644 --- a/res/app/components/stf/control/control-service.js +++ b/res/app/components/stf/control/control-service.js @@ -111,49 +111,8 @@ module.exports = function ControlServiceFactory( return sendTwoWay('device.identify') } - this.uploadUrl = function(url) { - var tx = TransactionService.create({ - id: 'storage' - }) - socket.emit('storage.upload', channel, tx.channel, { - url: url - }) - return tx.promise - } - - this.uploadFile = function(files) { - if (files.length !== 1) { - throw new Error('Can only upload one file') - } - var tx = TransactionService.create({ - id: 'storage' - }) - TransactionService.punch(tx.channel) - .then(function() { - $upload.upload({ - url: '/api/v1/resources?channel=' + tx.channel - , method: 'POST' - , file: files[0] - }) - }) - return tx.promise - } - this.install = function(options) { - var app = options.manifest.application - var params = { - url: options.url - } - if (app.launcherActivities.length) { - var activity = app.launcherActivities[0] - params.launchActivity = { - action: 'android.intent.action.MAIN' - , component: options.manifest.package + '/' + activity.name - , category: ['android.intent.category.LAUNCHER'] - , flags: 0x10200000 - } - } - return sendTwoWay('device.install', params) + return sendTwoWay('device.install', options) } this.uninstall = function(pkg) { diff --git a/res/app/components/stf/control/index.js b/res/app/components/stf/control/index.js index 5e5d01b3..5644cffa 100644 --- a/res/app/components/stf/control/index.js +++ b/res/app/components/stf/control/index.js @@ -3,3 +3,4 @@ module.exports = angular.module('stf/control', [ ]) .factory('TransactionService', require('./transaction-service')) .factory('ControlService', require('./control-service')) + .factory('StorageService', require('./storage-service')) diff --git a/res/app/components/stf/control/storage-service.js b/res/app/components/stf/control/storage-service.js new file mode 100644 index 00000000..bbc3d287 --- /dev/null +++ b/res/app/components/stf/control/storage-service.js @@ -0,0 +1,40 @@ +var Promise = require('bluebird') + +module.exports = function StorageServiceFactory($http, $upload) { + var service = {} + + service.storeUrl = function(type, url) { + return $http({ + url: '/api/v1/s/' + type + '/download' + , method: 'POST' + , data: { + url: url + } + }) + } + + service.storeFile = function(type, files) { + var resolver = Promise.defer() + + $upload.upload({ + url: '/api/v1/s/' + type + , method: 'POST' + , file: files + }) + .then( + function(value) { + resolver.resolve(value) + } + , function(err) { + resolver.reject(err) + } + , function(progressEvent) { + resolver.progress(progressEvent) + } + ) + + return resolver.promise + } + + return service +} diff --git a/res/app/control-panes/dashboard/upload/upload-controller.js b/res/app/control-panes/dashboard/upload/upload-controller.js index e0e87584..4407f5cb 100644 --- a/res/app/control-panes/dashboard/upload/upload-controller.js +++ b/res/app/control-panes/dashboard/upload/upload-controller.js @@ -1,5 +1,9 @@ -module.exports = function UploadCtrl($scope, SettingsService, gettext) { - +module.exports = function UploadCtrl( + $scope +, $http +, SettingsService +, StorageService +) { $scope.upload = null $scope.installation = null $scope.installEnabled = true @@ -35,23 +39,49 @@ module.exports = function UploadCtrl($scope, SettingsService, gettext) { $scope.installFile = function ($files) { $scope.upload = { - progress: 0, - lastData: 'uploading' + progress: 0 + , lastData: 'uploading' } - $scope.installation = null - return $scope.control.uploadFile($files) - .progressed(function (uploadResult) { - $scope.$apply(function () { - $scope.upload = uploadResult - }) + return StorageService.storeFile('apk', $files) + .progressed(function(e) { + if (e.lengthComputable) { + $scope.upload = { + progress: e.loaded / e.total * 100 + , lastData: 'uploading' + } + } }) - .then(function (uploadResult) { - $scope.$apply(function () { - $scope.upload = uploadResult - }) - if (uploadResult.success) { - return $scope.maybeInstall(uploadResult.body) + .then(function(res) { + $scope.upload = { + progress: 100 + , lastData: 'processing' + } + + var href = res.data.resources.file0.href + return $http.get(href + '/manifest') + .then(function(res) { + $scope.upload = { + progress: 100 + , lastData: 'success' + , settled: true + } + + if (res.data.success) { + return $scope.maybeInstall({ + href: href + , launch: $scope.launchEnabled + , manifest: res.data.manifest + }) + } + }) + }) + .catch(function(err) { + console.log('Upload error', err) + $scope.upload = { + progress: 100 + , lastData: 'fail' + , settled: true } }) } diff --git a/res/app/control-panes/dashboard/upload/upload.jade b/res/app/control-panes/dashboard/upload/upload.jade index 39b22f4b..7eab4302 100644 --- a/res/app/control-panes/dashboard/upload/upload.jade +++ b/res/app/control-panes/dashboard/upload/upload.jade @@ -4,7 +4,7 @@ span(translate) Upload clear-button(ng-click='clear()', ng-disabled='!installation && !upload').btn-xs label.checkbox-inline.pull-right - input(type='checkbox', ng-model='launchEnabled', ng-disabled='true') + input(type='checkbox', ng-model='launchEnabled') span Launch label.checkbox-inline.pull-right input(type='checkbox', ng-model='installEnabled') @@ -65,8 +65,6 @@ span(translate) Uploading... strong(ng-switch-when='processing') span(translate) Processing... - strong(ng-switch-when='storing') - span(translate) Storing... strong(ng-switch-when='fail') span(translate) Upload failed strong(ng-switch-when='success') diff --git a/webpack.config.js b/webpack.config.js index 67778886..3935dab0 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -26,7 +26,8 @@ module.exports = { , 'localforage': 'localforage/dist/localforage.js' , 'socket.io': 'socket.io-client/dist/socket.io' , 'oboe': 'oboe/dist/oboe-browser' - , 'ng-file-upload': 'ng-file-upload/angular-file-upload' + , 'ng-file-upload-shim5': 'ng-file-upload/angular-file-upload-html5-shim' + , 'ng-file-upload-main': 'ng-file-upload/angular-file-upload' , 'bluebird': 'bluebird/js/browser/bluebird' } }