diff --git a/lib/roles/app.js b/lib/roles/app.js index 8ba76876..3d197518 100644 --- a/lib/roles/app.js +++ b/lib/roles/app.js @@ -8,7 +8,6 @@ var validator = require('express-validator') var socketio = require('socket.io') var zmq = require('zmq') var Promise = require('bluebird') -var adb = require('adbkit') var logger = require('../util/logger') var pathutil = require('../util/pathutil') @@ -309,32 +308,6 @@ module.exports = function(options) { ]) }) - function fixedKeySender(klass, key) { - return function(channel) { - push.send([ - channel - , wireutil.envelope(new klass( - key - )) - ]) - } - } - - socket.on('input.back', fixedKeySender( - wire.KeyPressMessage - , adb.Keycode.KEYCODE_BACK - )) - - socket.on('input.home', fixedKeySender( - wire.KeyPressMessage - , adb.Keycode.KEYCODE_HOME - )) - - socket.on('input.menu', fixedKeySender( - wire.KeyPressMessage - , adb.Keycode.KEYCODE_MENU - )) - socket.on('flick', function(data) {}) socket.on('back', function(data) {}) socket.on('forward', function(data) {}) diff --git a/lib/roles/device.js b/lib/roles/device.js index faa24afb..502b7aa9 100644 --- a/lib/roles/device.js +++ b/lib/roles/device.js @@ -19,6 +19,8 @@ var pathutil = require('../util/pathutil') var promiseutil = require('../util/promiseutil') var Vitals = require('../util/vitals') var ChannelManager = require('../wire/channelmanager') +var keyutil = require('../util/keyutil') +var inputAgent = require('../services/inputagent') module.exports = function(options) { var log = logger.createLogger('device') @@ -37,7 +39,6 @@ module.exports = function(options) { } , services = { input: null - , monkey: null , logcat: null } @@ -137,8 +138,8 @@ module.exports = function(options) { , devutil.killProcsByComm( adb , options.serial - , 'commands.monkey' - , 'com.android.commands.monkey' + , 'app_process' + , 'app_process' ) ]) }) @@ -233,18 +234,19 @@ module.exports = function(options) { }) }) .then(function() { - log.info('Launching monkey service') - return devutil.ensureUnusedPort(adb, options.serial, 1080) + log.info('Launching InputAgent') + return devutil.ensureUnusedPort(adb, options.serial, 1090) .then(function(port) { - var log = logger.createLogger('device:remote:monkey') - return adb.shellAsync(options.serial, util.format( - // Some devices fail without an SD card installed; we can - // fake an external storage using this method - 'EXTERNAL_STORAGE=/data/local/tmp monkey --port %d' - , port - )) + var log = logger.createLogger('device:inputAgent') + return promiseutil.periodicNotify( + inputAgent.open(adb, options.serial) + , 1000 + ) + .progressed(function() { + log.info('Waiting for InputAgent') + }) .then(function(out) { - vitals.register('device:remote:monkey:shell', out) + vitals.register('device:inputAgent:shell', out) out.pipe(split()) .on('data', function(chunk) { log.info(chunk) @@ -256,12 +258,9 @@ module.exports = function(options) { return devutil.waitForPort(adb, options.serial, port) }) .then(function(conn) { - return monkey.connectStream(conn) - }) - .then(function(monkey) { - services.monkey = vitals.register( - 'device:remote:monkey:monkey' - , Promise.promisifyAll(monkey) + services.inputAgentSocket = vitals.register( + 'device:inputAgent:socket' + , conn ) }) }) @@ -378,29 +377,30 @@ module.exports = function(options) { log.error('tap failed', err.stack) }) }) - .on(wire.TypeMessage, function(channel, message) { - services.monkey.typeAsync(message.text) - .catch(function(err) { - log.error('type failed', err.stack) - }) - }) .on(wire.KeyDownMessage, function(channel, message) { - services.monkey.keyDownAsync(message.key) - .catch(function(err) { - log.error('keyDown failed', err.stack) - }) + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 0 + , keyCode: keyutil.unwire(message.keyCode) + }) }) .on(wire.KeyUpMessage, function(channel, message) { - services.monkey.keyUpAsync(message.key) - .catch(function(err) { - log.error('keyUp failed', err.stack) - }) + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 1 + , keyCode: keyutil.unwire(message.keyCode) + }) }) .on(wire.KeyPressMessage, function(channel, message) { - services.monkey.pressAsync(message.key) - .catch(function(err) { - log.error('keyPress failed', err.stack) - }) + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 2 + , keyCode: keyutil.unwire(message.keyCode) + }) + }) + .on(wire.TypeMessage, function(channel, message) { + inputAgent.sendInputEvent(services.inputAgentSocket, { + action: 3 + , keyCode: 0 + , text: message.text + }) }) .on(wire.LogcatApplyFiltersMessage, function(channel, message) { resetLogcat() diff --git a/lib/roles/provider.js b/lib/roles/provider.js index ec00274e..782e6d53 100644 --- a/lib/roles/provider.js +++ b/lib/roles/provider.js @@ -30,7 +30,7 @@ module.exports = function(options) { function totals() { if (lists.waiting.length) { log.info( - 'Providing %d of %d device(s), and still waiting for "%s"' + 'Providing %d of %d device(s); waiting for "%s"' , lists.ready.length , lists.all.length , lists.waiting.join('", "') @@ -38,10 +38,17 @@ module.exports = function(options) { delayedTotals() } + else if (lists.ready.length < lists.all.length) { + log.info( + 'Providing all %d of %d device(s); ignoring "%s"' + , lists.ready.length + , lists.all.length + , _.difference(lists.all, lists.ready).join('", "') + ) + } else { log.info( - 'Providing all %d of %d device(s)' - , lists.ready.length + 'Providing all %d device(s)' , lists.all.length ) } @@ -298,6 +305,8 @@ module.exports = function(options) { proc.on('error', errorListener) proc.on('message', messageListener) + lists.waiting.push(device.id) + return resolver.promise .finally(function() { log.info('Cleaning up device worker "%s"', device.id) diff --git a/lib/services/inputagent.js b/lib/services/inputagent.js new file mode 100644 index 00000000..b6a36128 --- /dev/null +++ b/lib/services/inputagent.js @@ -0,0 +1,48 @@ +var util = require('util') + +var Promise = require('bluebird') +var ProtoBuf = require('protobufjs') +var ByteBuffer = require('protobufjs/node_modules/bytebuffer') +var split = require('split') + +var pathutil = require('../util/pathutil') +var streamutil = require('../util/streamutil') + +var proto = ProtoBuf.loadProtoFile( + pathutil.vendor('InputAgent/inputAgentProtocol.proto') +).build().jp.co.cyberagent.stf.input.agent + +var inputAgent = module.exports = Object.create(null) + +inputAgent.open = function(adb, serial) { + return adb.installAsync(serial, pathutil.vendor('InputAgent/InputAgent.apk')) + .then(function() { + return adb.shellAsync(serial, 'pm path jp.co.cyberagent.stf.input.agent') + }) + .then(function(out) { + return streamutil.findLine(out, (/^package:/)) + .then(function(line) { + return line.substr(8) + }) + }) + .then(function(apk) { + return adb.shellAsync(serial, util.format( + "export CLASSPATH='%s';" + + ' exec app_process /system/bin' + + ' jp.co.cyberagent.stf.input.agent.InputAgent' + , apk + )) + }) +} + +inputAgent.sendInputEvent = function(agent, event) { + var lengthBuffer = new ByteBuffer() + , messageBuffer = new proto.InputEvent(event).encode() + + lengthBuffer.writeVarint32(messageBuffer.length) + + agent.write(Buffer.concat([ + lengthBuffer.toBuffer() + , messageBuffer.toBuffer() + ])) +} diff --git a/lib/util/devutil.js b/lib/util/devutil.js index 20f204aa..879c0096 100644 --- a/lib/util/devutil.js +++ b/lib/util/devutil.js @@ -92,6 +92,10 @@ devutil.waitForPort = function(adb, serial, port) { } devutil.listPidsByComm = function(adb, serial, comm, bin) { + var users = { + shell: true + } + return adb.shellAsync(serial, ['ps', comm]) .then(function(out) { return new Promise(function(resolve, reject) { @@ -104,7 +108,7 @@ devutil.listPidsByComm = function(adb, serial, comm, bin) { } else { var cols = chunk.toString().split(/\s+/) - if (cols.pop() === bin) { + if (cols.pop() === bin && users[cols[0]]) { pids.push(+cols[1]) } } diff --git a/lib/util/keyutil.js b/lib/util/keyutil.js index b90efcdb..4091a328 100644 --- a/lib/util/keyutil.js +++ b/lib/util/keyutil.js @@ -3,6 +3,8 @@ var util = require('util') var adb = require('adbkit') var Promise = require('bluebird') +var wire = require('../wire') + var keyutil = module.exports = Object.create(null) keyutil.parseKeyCharacterMap = function(stream) { @@ -460,7 +462,7 @@ keyutil.parseKeyCharacterMap = function(stream) { } keyutil.namedKey = function(name) { - var key = adb.Keycode['KEYCODE_' + name] + var key = adb.Keycode['KEYCODE_' + name.toUpperCase()] if (key === void 0) { throw new Error(util.format('Unknown key "%s"', name)) } @@ -570,3 +572,47 @@ keyutil.buildCharMap = function(keymap) { return charmap } + + +keyutil.unwire = (function() { + var map = Object.create(null) + + map[wire.KeyCode.HOME] = keyutil.namedKey('home') + map[wire.KeyCode.BACK] = keyutil.namedKey('back') + map[wire.KeyCode.BACKSPACE] = keyutil.namedKey('del') + map[wire.KeyCode.ENTER] = keyutil.namedKey('enter') + map[wire.KeyCode.CAPS_LOCK] = keyutil.namedKey('caps_lock') + map[wire.KeyCode.ESC] = keyutil.namedKey('escape') + map[wire.KeyCode.PAGE_UP] = keyutil.namedKey('page_up') + map[wire.KeyCode.PAGE_DOWN] = keyutil.namedKey('page_down') + map[wire.KeyCode.MOVE_END] = keyutil.namedKey('move_end') + map[wire.KeyCode.MOVE_HOME] = keyutil.namedKey('move_home') + map[wire.KeyCode.LEFT_ARROW] = keyutil.namedKey('dpad_left') + map[wire.KeyCode.UP_ARROW] = keyutil.namedKey('dpad_up') + map[wire.KeyCode.RIGHT_ARROW] = keyutil.namedKey('dpad_right') + map[wire.KeyCode.DOWN_ARROW] = keyutil.namedKey('dpad_down') + map[wire.KeyCode.INSERT] = keyutil.namedKey('insert') + map[wire.KeyCode.DELETE] = keyutil.namedKey('forward_del') + map[wire.KeyCode.MENU] = keyutil.namedKey('menu') + map[wire.KeyCode.F1] = keyutil.namedKey('f1') + map[wire.KeyCode.F2] = keyutil.namedKey('f2') + map[wire.KeyCode.F3] = keyutil.namedKey('f3') + map[wire.KeyCode.F4] = keyutil.namedKey('f4') + map[wire.KeyCode.F5] = keyutil.namedKey('f5') + map[wire.KeyCode.F6] = keyutil.namedKey('f6') + map[wire.KeyCode.F7] = keyutil.namedKey('f7') + map[wire.KeyCode.F8] = keyutil.namedKey('f8') + map[wire.KeyCode.F9] = keyutil.namedKey('f9') + map[wire.KeyCode.F10] = keyutil.namedKey('f10') + map[wire.KeyCode.F11] = keyutil.namedKey('f11') + map[wire.KeyCode.F12] = keyutil.namedKey('f12') + map[wire.KeyCode.NUM_LOCK] = keyutil.namedKey('num_lock') + + return function(keyCode) { + var key = map[keyCode] + if (!key) { + throw new Error(util.format('Unknown keycode "%s"', keyCode)) + } + return key + } +})() diff --git a/lib/util/promiseutil.js b/lib/util/promiseutil.js index 0d3482dd..12087755 100644 --- a/lib/util/promiseutil.js +++ b/lib/util/promiseutil.js @@ -8,12 +8,12 @@ module.exports.periodicNotify = function(promise, interval) { resolver.progress() } - function resolve() { - resolver.resolve() + function resolve(value) { + resolver.resolve(value) } - function reject() { - resolver.reject() + function reject(err) { + resolver.reject(err) } promise.then(resolve, reject) diff --git a/lib/util/streamutil.js b/lib/util/streamutil.js new file mode 100644 index 00000000..afc282d5 --- /dev/null +++ b/lib/util/streamutil.js @@ -0,0 +1,63 @@ +var Promise = require('bluebird') +var split = require('split') + +module.exports.readAll = function(stream) { + var resolver = Promise.defer() + , collected = new Buffer(0) + + function errorListener(err) { + resolver.reject(err) + } + + function endListener() { + resolver.resolve(collected) + } + + function readableListener() { + var chunk; + while (chunk = stream.read()) { + collected = Buffer.concat([collected, chunk]) + } + } + + stream.on('error', errorListener) + stream.on('readable', readableListener) + stream.on('end', endListener) + + return resolver.promise.finally(function() { + stream.removeListener('error', errorListener) + stream.removeListener('readable', readableListener) + stream.removeListener('end', endListener) + }) +} + + +module.exports.findLine = function(stream, re) { + var resolver = Promise.defer() + , piped = stream.pipe(split()) + + function errorListener(err) { + resolver.reject(err) + } + + function endListener() { + resolver.reject(new Error('No matching line found')) + } + + function lineListener(line) { + if (re.test(line)) { + resolver.resolve(line) + } + } + + piped.on('error', errorListener) + piped.on('data', lineListener) + piped.on('end', endListener) + + return resolver.promise.finally(function() { + piped.removeListener('error', errorListener) + piped.removeListener('data', lineListener) + piped.removeListener('end', endListener) + stream.unpipe(piped) + }) +} diff --git a/lib/wire/wire.proto b/lib/wire/wire.proto index 75db4495..71b30a0e 100644 --- a/lib/wire/wire.proto +++ b/lib/wire/wire.proto @@ -188,16 +188,49 @@ message TypeMessage { required string text = 1; } +enum KeyCode { + HOME = 3; + BACK = 4; + BACKSPACE = 8; + ENTER = 13; + CAPS_LOCK = 20; + ESC = 27; + PAGE_UP = 33; + PAGE_DOWN = 34; + MOVE_END = 35; + MOVE_HOME = 36; + LEFT_ARROW = 37; + UP_ARROW = 38; + RIGHT_ARROW = 39; + DOWN_ARROW = 40; + INSERT = 45; + DELETE = 46; + MENU = 93; + F1 = 112; + F2 = 113; + F3 = 114; + F4 = 115; + F5 = 116; + F6 = 117; + F7 = 118; + F8 = 119; + F9 = 120; + F10 = 121; + F11 = 122; + F12 = 123; + NUM_LOCK = 144; +} + message KeyDownMessage { - required uint32 key = 1; + required KeyCode keyCode = 1; } message KeyUpMessage { - required uint32 key = 1; + required KeyCode keyCode = 1; } message KeyPressMessage { - required uint32 key = 1; + required KeyCode keyCode = 1; } // Output diff --git a/res/app/scripts/controllers/DeviceScreenCtrl.js b/res/app/scripts/controllers/DeviceScreenCtrl.js new file mode 100644 index 00000000..d36fbe4b --- /dev/null +++ b/res/app/scripts/controllers/DeviceScreenCtrl.js @@ -0,0 +1,165 @@ +define(['./_module'], function(app) { + function DeviceScreenCtrl($scope, scalingService) { + $scope.ready = false + $scope.displayError = false + $scope.scalingService = scalingService + + $scope.promiseOfDevice.then(function() { + $scope.ready = true + }) + } + + function DeviceScreenDirective($document, scalingService) { + return { + restrict: 'E' + , templateUrl: 'partials/devices/screen' + , link: function($scope, element, attrs) { + $scope.promiseOfDevice.then(function(device) { + var loader = new Image() + , canvas = element.find('canvas')[0] + , finger = element.find('span') + , input = element.find('textarea') + , g = canvas.getContext('2d') + , displayWidth = 0 + , displayHeight = 0 + , scaler = scalingService.coordinator( + device.display.width + , device.display.height + ) + + function updateDisplaySize() { + displayWidth = element[0].offsetWidth + displayHeight = element[0].offsetHeight + + // Developer error, let's try to reduce debug time + if (!displayWidth || !displayHeight) { + throw new Error( + 'Unable to update display size; container must have dimensions' + ) + } + } + + function loadScreen() { + loader.src = device.display.url + + '?width=' + displayWidth + + '&height=' + displayHeight + + '&time=' + Date.now() + } + + loader.onload = function() { + var size = scaler.projectedSize(displayWidth, displayHeight) + + // Make sure we're rendering pixels 1 to 1 + canvas.width = this.width + canvas.height = this.height + + // Perhaps we have a massive screen but not enough pixels. Let's + // scale up + canvas.style.width = size.width + 'px' + canvas.style.height = size.height + 'px' + + // Draw the image + g.drawImage(this, 0, 0) + + // Reset error, if any + if ($scope.displayError) { + $scope.$apply(function() { + $scope.displayError = false + }) + } + + // Next please + loadScreen() + } + + loader.onerror = function() { + $scope.$apply(function() { + $scope.displayError = true + }) + } + + function sendTouch(type, e) { + var scaled = scaler.coords( + displayWidth + , displayHeight + , e.offsetX + , e.offsetY + ) + + finger[0].style.webkitTransform = + 'translate3d(' + e.offsetX + 'px,' + e.offsetY + 'px,0)' + + $scope.control[type]( + scaled.xP * device.display.width + , scaled.yP * device.display.height + ) + } + + function downListener(e) { + e.preventDefault() + input[0].focus() + element.addClass('fingering') + sendTouch('touchDown', e) + element.bind('mousemove', moveListener) + $document.bind('mouseup', upListener) + $document.bind('mouseleave', upListener) + } + + function moveListener(e) { + sendTouch('touchMove', e) + } + + function upListener(e) { + sendTouch('touchUp', e) + stop() + } + + function stop() { + element.removeClass('fingering') + element.unbind('mousemove', moveListener) + $document.unbind('mouseup', upListener) + $document.unbind('mouseleave', upListener) + } + + $scope.$on('$destroy', function() { + loader.onload = loader.onerror = null + stop() + }) + + input.bind('keydown', function(e) { + $scope.control.keyDown(e.keyCode) + }) + + input.bind('keyup', function(e) { + $scope.control.keyUp(e.keyCode) + }) + + input.bind('keypress', function(e) { + e.preventDefault() // no need to change value + $scope.control.type(String.fromCharCode(e.charCode)) + }) + + input.bind('paste', function(e) { + e.preventDefault() // no need to change value + $scope.control.type(e.clipboardData.getData('text/plain')) + }) + + element.bind('mousedown', downListener) + updateDisplaySize() + loadScreen() + }) + } + } + } + + app.controller('DeviceScreenCtrl' + , [ '$scope' + , 'ScalingService' + , DeviceScreenCtrl + ]) + .directive('deviceScreen' + , [ '$document' + , 'ScalingService' + , DeviceScreenDirective + ]) +}) diff --git a/res/app/scripts/services/ControlService.js b/res/app/scripts/services/ControlService.js new file mode 100644 index 00000000..d19c1acf --- /dev/null +++ b/res/app/scripts/services/ControlService.js @@ -0,0 +1,90 @@ +define(['./_module', 'lodash'], function(app, _) { + function ControlServiceFactory($rootScope, socket) { + var controlService = { + } + + function ControlService(channel) { + var keyCodes = { + 8: 8 // backspace + , 13: 13 // enter + , 20: 20 // caps lock + , 27: 27 // esc + , 33: 33 // page up + , 34: 34 // page down + , 35: 35 // end + , 36: 36 // home + , 37: 37 // left arrow + , 38: 38 // up arrow + , 39: 39 // right arrow + , 40: 40 // down arrow + , 45: 45 // insert + , 46: 46 // delete + , 93: 93 // windows menu key + , 112: 112 // f1 + , 113: 113 // f2 + , 114: 114 // f3 + , 115: 115 // f4 + , 116: 116 // f5 + , 117: 117 // f6 + , 118: 118 // f7 + , 119: 119 // f8 + , 120: 120 // f9 + , 121: 121 // f10 + , 122: 122 // f11 + , 123: 123 // f12 + , 144: 144 // num lock + } + + function touchSender(type) { + return function(x, y) { + socket.emit(type, channel, { + x: x + , y: y + }) + } + } + + function keySender(type, fixedKey) { + return function(key) { + var mapped = fixedKey || keyCodes[key] + if (mapped) { + socket.emit(type, channel, { + key: mapped + }) + } + } + } + + this.touchDown = touchSender('input.touchDown') + this.touchMove = touchSender('input.touchMove') + this.touchUp = touchSender('input.touchUp') + this.tap = touchSender('input.tap') + + this.keyDown = keySender('input.keyDown') + this.keyUp = keySender('input.keyUp') + this.keyPress = keySender('input.keyPress') + + this.home = keySender('input.keyPress', 3) + this.menu = keySender('input.keyPress', 93) + this.back = keySender('input.keyPress', 4) + + this.type = function(text) { + socket.emit('input.type', channel, { + text: text + }) + } + } + + controlService.forChannel = function(channel) { + return new ControlService(channel) + } + + return controlService + } + + app.factory('ControlService' + , [ '$rootScope' + , 'SocketService' + , ControlServiceFactory + ]) +}) diff --git a/vendor/InputAgent/InputAgent.apk b/vendor/InputAgent/InputAgent.apk new file mode 100644 index 00000000..05933e65 Binary files /dev/null and b/vendor/InputAgent/InputAgent.apk differ diff --git a/vendor/InputAgent/inputAgentProtocol.proto b/vendor/InputAgent/inputAgentProtocol.proto new file mode 100644 index 00000000..300f39f4 --- /dev/null +++ b/vendor/InputAgent/inputAgentProtocol.proto @@ -0,0 +1,23 @@ +package jp.co.cyberagent.stf.input.agent; + +enum InputAction { + KEYDOWN = 0; + KEYUP = 1; + KEYPRESS = 2; + TYPE = 3; +} + +message InputEvent { + required InputAction action = 1; + required int32 keyCode = 2; + optional bool shiftKey = 3; + optional bool ctrlKey = 4; + optional bool altKey = 5; + optional bool metaKey = 6; + optional bool symKey = 7; + optional bool functionKey = 8; + optional bool capsLockKey = 9; + optional bool scrollLockKey = 10; + optional bool numLockKey = 11; + optional string text = 12; +}