From 87e80ecb00deb5f3c1695c3b71d7b5ed9fc38686 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 7 Sep 2015 10:49:41 +0900 Subject: [PATCH 01/23] Base structure for vnc plugin. --- lib/cli.js | 8 ++++++++ lib/units/device/index.js | 1 + lib/units/device/plugins/vnc.js | 10 ++++++++++ lib/units/provider/index.js | 2 +- 4 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 lib/units/device/plugins/vnc.js diff --git a/lib/cli.js b/lib/cli.js index 9b0ed5c9..a10a6e44 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -103,6 +103,7 @@ program , '--connect-push', options.connectPush.join(',') , '--screen-port', ports.shift() , '--connect-port', ports.shift() + , '--vnc-port', ports.shift() , '--public-ip', options.publicIp , '--group-timeout', options.groupTimeout , '--storage-url', options.storageUrl @@ -142,6 +143,9 @@ program .option('--connect-port ' , 'port allocated to adb connect' , Number) + .option('--vnc-port ' + , 'port allocated to vnc' + , Number) .option('--connect-url-pattern ' , 'adb connect URL pattern' , String @@ -193,6 +197,9 @@ program if (!options.connectPort) { this.missingArgument('--connect-port') } + if (!options.vncPort) { + this.missingArgument('--vnc-port') + } if (!options.storageUrl) { this.missingArgument('--storage-url') } @@ -213,6 +220,7 @@ program , screenPort: options.screenPort , connectUrlPattern: options.connectUrlPattern , connectPort: options.connectPort + , vncPort: options.vncPort , heartbeatInterval: options.heartbeatInterval , muteMaster: options.muteMaster , lockRotation: options.lockRotation diff --git a/lib/units/device/index.js b/lib/units/device/index.js index fa2a3a29..26fbafb0 100644 --- a/lib/units/device/index.js +++ b/lib/units/device/index.js @@ -20,6 +20,7 @@ module.exports = function(options) { .dependency(require('./plugins/solo')) .dependency(require('./plugins/screen/stream')) .dependency(require('./plugins/screen/capture')) + .dependency(require('./plugins/vnc')) .dependency(require('./plugins/service')) .dependency(require('./plugins/browser')) .dependency(require('./plugins/store')) diff --git a/lib/units/device/plugins/vnc.js b/lib/units/device/plugins/vnc.js new file mode 100644 index 00000000..51b93286 --- /dev/null +++ b/lib/units/device/plugins/vnc.js @@ -0,0 +1,10 @@ +var syrup = require('stf-syrup') +var Promise = require('bluebird') +var _ = require('lodash') + +var logger = require('../../../util/logger') + +module.exports = syrup.serial() + .define(function(options) { + + }) diff --git a/lib/units/provider/index.js b/lib/units/provider/index.js index 981875a0..3d196b41 100644 --- a/lib/units/provider/index.js +++ b/lib/units/provider/index.js @@ -316,7 +316,7 @@ module.exports = function(options) { // Spawn a device worker function spawn() { - var allocatedPorts = ports.splice(0, 2) + var allocatedPorts = ports.splice(0, 4) , proc = options.fork(device, allocatedPorts) , resolver = Promise.defer() From 045737e4bf66d09f22a14d72e213d25a4da97372 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 7 Sep 2015 10:51:25 +0900 Subject: [PATCH 02/23] VNC plugin should have its own folder since it'll need supporting files. --- lib/units/device/plugins/{vnc.js => vnc/index.js} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/units/device/plugins/{vnc.js => vnc/index.js} (100%) diff --git a/lib/units/device/plugins/vnc.js b/lib/units/device/plugins/vnc/index.js similarity index 100% rename from lib/units/device/plugins/vnc.js rename to lib/units/device/plugins/vnc/index.js From 66182666bd757937adaf678c19d7da48aaed7182 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Sun, 13 Sep 2015 20:09:15 +0900 Subject: [PATCH 03/23] Now able to read VNC Viewer's primary requests. --- .../device/plugins/vnc/util/pixelformat.js | 14 + .../device/plugins/vnc/util/vncserver.js | 397 ++++++++++++++++++ package.json | 1 + 3 files changed, 412 insertions(+) create mode 100644 lib/units/device/plugins/vnc/util/pixelformat.js create mode 100644 lib/units/device/plugins/vnc/util/vncserver.js diff --git a/lib/units/device/plugins/vnc/util/pixelformat.js b/lib/units/device/plugins/vnc/util/pixelformat.js new file mode 100644 index 00000000..9a1c4273 --- /dev/null +++ b/lib/units/device/plugins/vnc/util/pixelformat.js @@ -0,0 +1,14 @@ +function PixelFormat(values) { + this.bitsPerPixel = values.bitsPerPixel + this.depth = values.depth + this.bigEndianFlag = values.bigEndianFlag + this.trueColorFlag = values.trueColorFlag + this.redMax = values.redMax + this.greenMax = values.greenMax + this.blueMax = values.blueMax + this.redShift = values.redShift + this.greenShift = values.greenShift + this.blueShift = values.blueShift +} + +module.exports = PixelFormat diff --git a/lib/units/device/plugins/vnc/util/vncserver.js b/lib/units/device/plugins/vnc/util/vncserver.js new file mode 100644 index 00000000..f884b647 --- /dev/null +++ b/lib/units/device/plugins/vnc/util/vncserver.js @@ -0,0 +1,397 @@ +var net = require('net') +var util = require('util') + +var EventEmitter = require('eventemitter3').EventEmitter +var debug = require('debug')('vnc') + +var PixelFormat = require('./pixelformat') + +function VncServer(server) { + this._bound = { + _connectionListener: this._connectionListener.bind(this) + } + + this.server = server + .on('connection', this._bound._connectionListener) +} + +util.inherits(VncServer, EventEmitter) + +VncServer.prototype._connectionListener = function(conn) { + debug('connection', conn.remoteAddress, conn.remotePort) + new VncConnection(conn) +} + +function VncConnection(conn) { + this._bound = { + _readableListener: this._readableListener.bind(this) + } + + this._buffer = null + this._state = 0 + this._changeState(VncConnection.STATE_NEED_CLIENT_VERSION) + + this._serverVersion = VncConnection.V3_008 + this._serverSupportedSecurity = [VncConnection.SECURITY_NONE] + this._serverWidth = 800 + this._serverHeight = 600 + this._serverPixelFormat = new PixelFormat({ + bitsPerPixel: 32 + , depth: 24 + , bigEndianFlag: 1 + , trueColorFlag: 1 + , redMax: 255 + , greenMax: 255 + , blueMax: 255 + , redShift: 16 + , greenShift: 8 + , blueShift: 0 + }) + this._serverName = 'stf' + + this._clientVersion = null + this._clientShare = false + this._clientWidth = this._serverWidth + this._clientHeight = this._serverHeight + this._clientPixelFormat = this._serverPixelFormat + this._clientEncodingCount = 0 + this._clientEncodings = [] + + this.conn = conn + .on('readable', this._bound._readableListener) + + this._writeServerVersion() + this._read() +} + +util.inherits(VncConnection, EventEmitter) + +VncConnection.V3_003 = 3003 +VncConnection.V3_007 = 3007 +VncConnection.V3_008 = 3008 + +VncConnection.SECURITY_NONE = 1 +VncConnection.SECURITY_VNC = 2 + +VncConnection.SECURITYRESULT_OK = 0 +VncConnection.SECURITYRESULT_FAIL = 1 + +VncConnection.CLIENT_MESSAGE_SETPIXELFORMAT = 0 +VncConnection.CLIENT_MESSAGE_SETENCODINGS = 2 +VncConnection.CLIENT_MESSAGE_FBUPDATEREQUEST = 3 +VncConnection.CLIENT_MESSAGE_KEYEVENT = 4 +VncConnection.CLIENT_MESSAGE_POINTEREVENT = 5 +VncConnection.CLIENT_MESSAGE_CLIENTCUTTEXT = 6 + +var StateReverse = Object.create(null), State = { + STATE_NEED_CLIENT_VERSION: 10 +, STATE_NEED_CLIENT_SECURITY: 20 +, STATE_NEED_CLIENT_INIT: 30 +, STATE_NEED_CLIENT_MESSAGE: 40 +, STATE_NEED_CLIENT_MESSAGE_SETPIXELFORMAT: 50 +, STATE_NEED_CLIENT_MESSAGE_SETENCODINGS: 60 +, STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_ARRAY: 61 +, STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST: 70 +, STATE_NEED_CLIENT_MESSAGE_KEYEVENT: 80 +, STATE_NEED_CLIENT_MESSAGE_POINTEREVENT: 90 +, STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT: 100 +} + +Object.keys(State).map(function(name) { + VncConnection[name] = State[name] + StateReverse[State[name]] = name +}) + +VncConnection.prototype._writeServerVersion = function() { + // Yes, we could just format the string instead. Didn't feel like it. + switch (this._serverVersion) { + case VncConnection.V3_003: + this._write(new Buffer('RFB 003.003\n')) + break + case VncConnection.V3_007: + this._write(new Buffer('RFB 003.007\n')) + break + case VncConnection.V3_008: + this._write(new Buffer('RFB 003.008\n')) + break + } +} + +VncConnection.prototype._writeSupportedSecurity = function() { + var chunk = new Buffer(1 + this._serverSupportedSecurity.length) + + chunk[0] = this._serverSupportedSecurity.length + this._serverSupportedSecurity.forEach(function(security, i) { + chunk[1 + i] = security + }) + + this._write(chunk) +} + +VncConnection.prototype._writeSelectedSecurity = function() { + var chunk = new Buffer(4) + chunk.writeUInt32BE(VncConnection.SECURITY_NONE, 0) + this._write(chunk) +} + +VncConnection.prototype._writeSecurityResult = function(result, reason) { + var chunk + switch (result) { + case VncConnection.SECURITYRESULT_OK: + chunk = new Buffer(4) + chunk.writeUInt32BE(result, 0) + this._write(chunk) + break + case VncConnection.SECURITYRESULT_FAIL: + chunk = new Buffer(4 + 4 + reason.length) + chunk.writeUInt32BE(result, 0) + chunk.writeUInt32BE(reason.length, 4) + chunk.write(reason, 8, reason.length) + this._write(chunk) + break + } +} + +VncConnection.prototype._writeServerInit = function() { + var chunk = new Buffer(2 + 2 + 16 + 4 + this._serverName.length) + chunk.writeUInt16BE(this._serverWidth, 0) + chunk.writeUInt16BE(this._serverHeight, 2) + chunk[4] = this._serverPixelFormat.bitsPerPixel + chunk[5] = this._serverPixelFormat.depth + chunk[6] = this._serverPixelFormat.bigEndianFlag + chunk[7] = this._serverPixelFormat.trueColorFlag + chunk.writeUInt16BE(this._serverPixelFormat.redMax, 8) + chunk.writeUInt16BE(this._serverPixelFormat.greenMax, 10) + chunk.writeUInt16BE(this._serverPixelFormat.blueMax, 12) + chunk[14] = this._serverPixelFormat.redShift + chunk[15] = this._serverPixelFormat.greenShift + chunk[16] = this._serverPixelFormat.blueShift + chunk[17] = 0 // padding + chunk[18] = 0 // padding + chunk[19] = 0 // padding + chunk.writeUInt32BE(this._serverName.length, 20) + chunk.write(this._serverName, 24, this._serverName.length) + this._write(chunk) +} + +VncConnection.prototype._readableListener = function() { + this._read() +} + +VncConnection.prototype._read = function() { + var chunk, lo, hi + while (this._append(this.conn.read())) { + do { + debug('state', StateReverse[this._state]) + switch (this._state) { + case VncConnection.STATE_NEED_CLIENT_VERSION: + if ((chunk = this._consume(12))) { + this._clientVersion = this._parseVersion(chunk) + debug('client version', this._clientVersion) + this._writeSupportedSecurity() + this._changeState(VncConnection.STATE_NEED_CLIENT_SECURITY) + } + break + case VncConnection.STATE_NEED_CLIENT_SECURITY: + if ((chunk = this._consume(1))) { + this._clientSecurity = this._parseSecurity(chunk) + debug('client security', this._clientSecurity) + this._writeSecurityResult(VncConnection.SECURITYRESULT_OK) + this._changeState(VncConnection.STATE_NEED_CLIENT_INIT) + } + break + case VncConnection.STATE_NEED_CLIENT_INIT: + if ((chunk = this._consume(1))) { + this._clientShare = chunk[0] + debug('client shareFlag', this._clientShare) + this._writeServerInit() + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE: + if ((chunk = this._consume(1))) { + switch (chunk[0]) { + case VncConnection.CLIENT_MESSAGE_SETPIXELFORMAT: + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETPIXELFORMAT) + break + case VncConnection.CLIENT_MESSAGE_SETENCODINGS: + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS) + break + case VncConnection.CLIENT_MESSAGE_FBUPDATEREQUEST: + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST) + break + case VncConnection.CLIENT_MESSAGE_KEYEVENT: + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_KEYEVENT) + break + case VncConnection.CLIENT_MESSAGE_POINTEREVENT: + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_POINTEREVENT) + break + case VncConnection.CLIENT_MESSAGE_CLIENTCUTTEXT: + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT) + break + default: + throw new Error(util.format('Unsupported message type %d', chunk[0])) + } + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_SETPIXELFORMAT: + if ((chunk = this._consume(19))) { + // [0b, 3b) padding + this._clientPixelFormat = new PixelFormat({ + bitsPerPixel: chunk[3] + , depth: chunk[4] + , bigEndianFlag: chunk[5] + , trueColorFlag: chunk[6] + , redMax: chunk.readUInt16BE(7, true) + , greenMax: chunk.readUInt16BE(9, true) + , blueMax: chunk.readUInt16BE(11, true) + , redShift: chunk[13] + , greenShift: chunk[14] + , blueShift: chunk[15] + }) + // [16b, 19b) padding + debug('client pixel format', this._clientPixelFormat) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS: + if ((chunk = this._consume(3))) { + // [0b, 1b) padding + this._clientEncodingCount = chunk.readUInt16BE(1, true) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_ARRAY) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_ARRAY: + lo = 0 + hi = 4 * this._clientEncodingCount + if ((chunk = this._consume(hi))) { + this._clientEncodings = [] + while (lo < hi) { + this._clientEncodings.push(chunk.readInt32BE(lo, true)) + lo += 4 + } + debug('client encodings', this._clientEncodings) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST: + if ((chunk = this._consume(9))) { + // incremental = chunk[0] + // xPosition = chunk.readUInt16BE(1, true) + // yPosition = chunk.readUInt16BE(3, true) + // width = chunk.readUInt16BE(5, true) + // height = chunk.readUInt16BE(7, true) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_KEYEVENT: + if ((chunk = this._consume(7))) { + // downFlag = chunk[0] + // [1b, 3b) padding + // key = chunk.readUInt32BE(3, true) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_POINTEREVENT: + if ((chunk = this._consume(5))) { + // buttonMask = chunk[0] + // xPosition = chunk.readUInt16BE(1, true) + // yPosition = chunk.readUInt16BE(3, true) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT: + if ((chunk = this._consume(7))) { + // [0b, 3b) padding + // length = chunk.readUInt32BE(3) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE) + } + break + } + } + while (chunk) + } +} + +VncConnection.prototype._parseVersion = function(chunk) { + if (chunk.equals(new Buffer('RFB 003.008\n'))) { + return VncConnection.V3_008 + } + + if (chunk.equals(new Buffer('RFB 003.007\n'))) { + return VncConnection.V3_007 + } + + if (chunk.equals(new Buffer('RFB 003.003\n'))) { + return VncConnection.V3_003 + } + + throw new Error('Unsupported version') +} + +VncConnection.prototype._parseSecurity = function(chunk) { + switch (chunk[0]) { + case VncConnection.SECURITY_NONE: + case VncConnection.SECURITY_VNC: + return chunk[0] + default: + throw new Error('Unsupported security type') + } +} + +VncConnection.prototype._changeState = function(state) { + this._state = state +} + +VncConnection.prototype._append = function(chunk) { + if (!chunk) { + return false + } + + debug('in', chunk) + + if (this._buffer) { + this._buffer = Buffer.concat( + [this._buffer, chunk], this._buffer.length + chunk.length) + } + else { + this._buffer = chunk + } + + return true +} + +VncConnection.prototype._consume = function(n) { + var chunk + + if (!this._buffer) { + return null + } + + if (n < this._buffer.length) { + chunk = this._buffer.slice(0, n) + this._buffer = this._buffer.slice(n) + return chunk + } + + if (n === this._buffer.length) { + chunk = this._buffer + this._buffer = null + return chunk + } + + return null +} + +VncConnection.prototype._write = function(chunk) { + debug('out', chunk) + this.conn.write(chunk) +} + +var nserv = net.createServer({ + allowHalfOpen: true +}) + +var vserv = new VncServer(nserv) + +nserv.listen(5910) diff --git a/package.json b/package.json index a00ce0a1..879024d1 100644 --- a/package.json +++ b/package.json @@ -40,6 +40,7 @@ "compression": "^1.5.2", "cookie-session": "^1.2.0", "csurf": "^1.7.0", + "debug": "^2.2.0", "eventemitter3": "^0.1.6", "express": "^4.13.3", "express-validator": "^2.17.1", From 6d84064e6bb198f78adb1382bb4649dc92c798be Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 13:28:26 +0900 Subject: [PATCH 04/23] Consume ClientCutText's value. --- .../device/plugins/vnc/util/vncserver.js | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/units/device/plugins/vnc/util/vncserver.js b/lib/units/device/plugins/vnc/util/vncserver.js index f884b647..129f4aae 100644 --- a/lib/units/device/plugins/vnc/util/vncserver.js +++ b/lib/units/device/plugins/vnc/util/vncserver.js @@ -56,6 +56,7 @@ function VncConnection(conn) { this._clientPixelFormat = this._serverPixelFormat this._clientEncodingCount = 0 this._clientEncodings = [] + this._clientCutTextLength = 0 this.conn = conn .on('readable', this._bound._readableListener) @@ -90,11 +91,12 @@ var StateReverse = Object.create(null), State = { , STATE_NEED_CLIENT_MESSAGE: 40 , STATE_NEED_CLIENT_MESSAGE_SETPIXELFORMAT: 50 , STATE_NEED_CLIENT_MESSAGE_SETENCODINGS: 60 -, STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_ARRAY: 61 +, STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_VALUE: 61 , STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST: 70 , STATE_NEED_CLIENT_MESSAGE_KEYEVENT: 80 , STATE_NEED_CLIENT_MESSAGE_POINTEREVENT: 90 , STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT: 100 +, STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE: 101 } Object.keys(State).map(function(name) { @@ -258,10 +260,10 @@ VncConnection.prototype._read = function() { if ((chunk = this._consume(3))) { // [0b, 1b) padding this._clientEncodingCount = chunk.readUInt16BE(1, true) - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_ARRAY) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_VALUE) } break - case VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_ARRAY: + case VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_VALUE: lo = 0 hi = 4 * this._clientEncodingCount if ((chunk = this._consume(hi))) { @@ -303,10 +305,18 @@ VncConnection.prototype._read = function() { case VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT: if ((chunk = this._consume(7))) { // [0b, 3b) padding - // length = chunk.readUInt32BE(3) - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE) + this._clientCutTextLength = chunk.readUInt32BE(3) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_STRING) } break + case VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE: + if ((chunk = this._consume(this._clientCutTextLength))) { + // value = chunk + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) + } + break + default: + throw new Error(util.format('Impossible state %d', this._state)) } } while (chunk) From f7d1b07d1c3cc534b510f3880242a7bdfc7f695c Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 14:09:45 +0900 Subject: [PATCH 05/23] Clean up structure a little bit for initial testing. --- .../vnc/util/{vncserver.js => connection.js} | 27 ++----------------- lib/units/device/plugins/vnc/util/example.js | 10 +++++++ lib/units/device/plugins/vnc/util/server.js | 25 +++++++++++++++++ 3 files changed, 37 insertions(+), 25 deletions(-) rename lib/units/device/plugins/vnc/util/{vncserver.js => connection.js} (95%) create mode 100644 lib/units/device/plugins/vnc/util/example.js create mode 100644 lib/units/device/plugins/vnc/util/server.js diff --git a/lib/units/device/plugins/vnc/util/vncserver.js b/lib/units/device/plugins/vnc/util/connection.js similarity index 95% rename from lib/units/device/plugins/vnc/util/vncserver.js rename to lib/units/device/plugins/vnc/util/connection.js index 129f4aae..cc1cc3d7 100644 --- a/lib/units/device/plugins/vnc/util/vncserver.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -1,27 +1,10 @@ -var net = require('net') var util = require('util') var EventEmitter = require('eventemitter3').EventEmitter -var debug = require('debug')('vnc') +var debug = require('debug')('vnc:connection') var PixelFormat = require('./pixelformat') -function VncServer(server) { - this._bound = { - _connectionListener: this._connectionListener.bind(this) - } - - this.server = server - .on('connection', this._bound._connectionListener) -} - -util.inherits(VncServer, EventEmitter) - -VncServer.prototype._connectionListener = function(conn) { - debug('connection', conn.remoteAddress, conn.remotePort) - new VncConnection(conn) -} - function VncConnection(conn) { this._bound = { _readableListener: this._readableListener.bind(this) @@ -398,10 +381,4 @@ VncConnection.prototype._write = function(chunk) { this.conn.write(chunk) } -var nserv = net.createServer({ - allowHalfOpen: true -}) - -var vserv = new VncServer(nserv) - -nserv.listen(5910) +module.exports = VncConnection diff --git a/lib/units/device/plugins/vnc/util/example.js b/lib/units/device/plugins/vnc/util/example.js new file mode 100644 index 00000000..6ed338e0 --- /dev/null +++ b/lib/units/device/plugins/vnc/util/example.js @@ -0,0 +1,10 @@ +var net = require('net') +var VncServer = require('./server') + +var nserv = net.createServer({ + allowHalfOpen: true +}) + +var vserv = new VncServer(nserv) + +nserv.listen(5910) diff --git a/lib/units/device/plugins/vnc/util/server.js b/lib/units/device/plugins/vnc/util/server.js new file mode 100644 index 00000000..2c8f6162 --- /dev/null +++ b/lib/units/device/plugins/vnc/util/server.js @@ -0,0 +1,25 @@ +var net = require('net') +var util = require('util') + +var EventEmitter = require('eventemitter3').EventEmitter +var debug = require('debug')('vnc:server') + +var VncConnection = require('./connection') + +function VncServer(server) { + this._bound = { + _connectionListener: this._connectionListener.bind(this) + } + + this.server = server + .on('connection', this._bound._connectionListener) +} + +util.inherits(VncServer, EventEmitter) + +VncServer.prototype._connectionListener = function(conn) { + debug('connection', conn.remoteAddress, conn.remotePort) + new VncConnection(conn) +} + +module.exports = VncServer From 8a5f0551a75537892d8c21236931270c908d98b6 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 14:40:19 +0900 Subject: [PATCH 06/23] Fix incorrect state change for ClientCutText. --- lib/units/device/plugins/vnc/util/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index cc1cc3d7..8e55fa4b 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -289,7 +289,7 @@ VncConnection.prototype._read = function() { if ((chunk = this._consume(7))) { // [0b, 3b) padding this._clientCutTextLength = chunk.readUInt32BE(3) - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_STRING) + this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE) } break case VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE: From 792713d41559c6cc64105b6d965a57a2062d1784 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 18:00:17 +0900 Subject: [PATCH 07/23] VNC screen is visible (w/ RAW encoding). Size of VNC screen is still hardcoded, preventing real use. --- lib/units/device/plugins/screen/stream.js | 121 ++++++++-------- lib/units/device/plugins/vnc/index.js | 121 +++++++++++++++- .../device/plugins/vnc/util/connection.js | 129 +++++++++++++++--- lib/units/device/plugins/vnc/util/example.js | 10 -- lib/units/device/plugins/vnc/util/server.js | 31 ++++- 5 files changed, 320 insertions(+), 92 deletions(-) delete mode 100644 lib/units/device/plugins/vnc/util/example.js diff --git a/lib/units/device/plugins/screen/stream.js b/lib/units/device/plugins/screen/stream.js index f75069fa..bc9e5b7d 100644 --- a/lib/units/device/plugins/screen/stream.js +++ b/lib/units/device/plugins/screen/stream.js @@ -443,7 +443,7 @@ module.exports = syrup.serial() return createServer() .then(function(wss) { - var broadcastSet = new BroadcastSet() + var broadcastSet = plugin.broadcastSet = new BroadcastSet() var frameProducer = new FrameProducer( new FrameConfig(display.properties, display.properties)) @@ -460,32 +460,8 @@ module.exports = syrup.serial() }) frameProducer.on('start', function() { - var message = util.format( - 'start %s' - , JSON.stringify(frameProducer.banner) - ) - - broadcastSet.keys().forEach(function(id) { - var ws = broadcastSet.get(id) - switch (ws.readyState) { - case WebSocket.OPENING: - // This should never happen. - log.warn('Unable to send banner to OPENING client "%s"', id) - break - case WebSocket.OPEN: - // This is what SHOULD happen. - ws.send(message) - break - case WebSocket.CLOSING: - // Ok, a 'close' event should remove the client from the set - // soon. - break - case WebSocket.CLOSED: - // This should never happen. - log.warn('Unable to send banner to CLOSED client "%s"', id) - broadcastSet.remove(id) - break - } + broadcastSet.keys().map(function(id) { + return broadcastSet.get(id).onStart(frameProducer) }) }) @@ -493,32 +469,7 @@ module.exports = syrup.serial() var frame if ((frame = frameProducer.nextFrame())) { Promise.settle([broadcastSet.keys().map(function(id) { - return new Promise(function(resolve, reject) { - var ws = broadcastSet.get(id) - switch (ws.readyState) { - case WebSocket.OPENING: - // This should never happen. - return reject(new Error(util.format( - 'Unable to send frame to OPENING client "%s"', id))) - case WebSocket.OPEN: - // This is what SHOULD happen. - ws.send(frame, { - binary: true - }, function(err) { - return err ? reject(err) : resolve() - }) - return - case WebSocket.CLOSING: - // Ok, a 'close' event should remove the client from the set - // soon. - return - case WebSocket.CLOSED: - // This should never happen. - broadcastSet.remove(id) - return reject(new Error(util.format( - 'Unable to send frame to CLOSED client "%s"', id))) - } - }) + return broadcastSet.get(id).onFrame(frame) })]).then(next) } else { @@ -534,12 +485,74 @@ module.exports = syrup.serial() wss.on('connection', function(ws) { var id = uuid.v4() + function wsStartNotifier() { + return new Promise(function(resolve, reject) { + var message = util.format( + 'start %s' + , JSON.stringify(frameProducer.banner) + ) + + switch (ws.readyState) { + case WebSocket.OPENING: + // This should never happen. + log.warn('Unable to send banner to OPENING client "%s"', id) + break + case WebSocket.OPEN: + // This is what SHOULD happen. + ws.send(message, function(err) { + return err ? reject(err) : resolve() + }) + break + case WebSocket.CLOSING: + // Ok, a 'close' event should remove the client from the set + // soon. + break + case WebSocket.CLOSED: + // This should never happen. + log.warn('Unable to send banner to CLOSED client "%s"', id) + broadcastSet.remove(id) + break + } + }) + } + + function wsFrameNotifier(frame) { + return new Promise(function(resolve, reject) { + switch (ws.readyState) { + case WebSocket.OPENING: + // This should never happen. + return reject(new Error(util.format( + 'Unable to send frame to OPENING client "%s"', id))) + case WebSocket.OPEN: + // This is what SHOULD happen. + ws.send(frame, { + binary: true + }, function(err) { + return err ? reject(err) : resolve() + }) + return + case WebSocket.CLOSING: + // Ok, a 'close' event should remove the client from the set + // soon. + return + case WebSocket.CLOSED: + // This should never happen. + broadcastSet.remove(id) + return reject(new Error(util.format( + 'Unable to send frame to CLOSED client "%s"', id))) + } + }) + } + ws.on('message', function(data) { var match if ((match = /^(on|off|(size) ([0-9]+)x([0-9]+))$/.exec(data))) { switch (match[2] || match[1]) { case 'on': - broadcastSet.insert(id, ws) + broadcastSet.insert(id, { + onStart: wsStartNotifier + , onFrame: wsFrameNotifier + }) break case 'off': broadcastSet.remove(id) diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 51b93286..66581f3b 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -1,10 +1,125 @@ +var net = require('net') +var util = require('util') + var syrup = require('stf-syrup') var Promise = require('bluebird') -var _ = require('lodash') +var uuid = require('node-uuid') +var jpeg = require('jpeg-js') -var logger = require('../../../util/logger') +var logger = require('../../../../util/logger') +var lifecycle = require('../../../../util/lifecycle') + +var VncServer = require('./util/server') +var VncConnection = require('./util/connection') module.exports = syrup.serial() - .define(function(options) { + .dependency(require('../screen/stream')) + .define(function(options, screenStream) { + var log = logger.createLogger('device:plugins:vnc') + function createServer() { + log.info('Starting VNC server on port %d', options.vncPort) + + var vnc = new VncServer(net.createServer({ + allowHalfOpen: true + })) + + var listeningListener, errorListener + return new Promise(function(resolve, reject) { + listeningListener = function() { + return resolve(vnc) + } + + errorListener = function(err) { + return reject(err) + } + + vnc.on('listening', listeningListener) + vnc.on('error', errorListener) + + vnc.listen(options.vncPort) + }) + .finally(function() { + vnc.removeListener('listening', listeningListener) + vnc.removeListener('error', errorListener) + }) + } + + return createServer() + .then(function(vnc) { + vnc.on('connection', function(conn) { + var id = util.format('vnc-%s', uuid.v4()) + + var connState = { + lastFrame: null + , lastFrameTime: null + , frameWidth: 0 + , frameHeight: 0 + , sentFrameTime: null + , updateRequests: 0 + } + + function vncStartListener(frameProducer) { + return new Promise(function(resolve/*, reject*/) { + connState.frameWidth = frameProducer.banner.virtualWidth + connState.frameHeight = frameProducer.banner.virtualHeight + resolve() + }) + } + + function vncFrameListener(frame) { + return new Promise(function(resolve/*, reject*/) { + connState.lastFrame = frame + connState.lastFrameTime = Date.now() + maybeSendFrame() + resolve() + }) + } + + function maybeSendFrame() { + if (!connState.updateRequests) { + return + } + + if (!connState.lastFrame) { + return + } + + if (connState.lastFrameTime === connState.sentFrameTime) { + return + } + + var decoded = jpeg.decode(connState.lastFrame) + conn.writeFramebufferUpdate([{ + xPosition: 0 + , yPosition: 0 + , width: connState.frameWidth + , height: connState.frameHeight + , encodingType: VncConnection.ENCODING_RAW + , data: decoded.data + }]) + + connState.updateRequests = 0 + connState.sentFrameTime = connState.lastFrameTime + } + + screenStream.broadcastSet.insert(id, { + onStart: vncStartListener + , onFrame: vncFrameListener + }) + + conn.on('fbupdaterequest', function() { + connState.updateRequests += 1 + maybeSendFrame() + }) + + conn.on('close', function() { + screenStream.broadcastSet.remove(id) + }) + }) + + lifecycle.observe(function() { + vnc.close() + }) + }) }) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 8e55fa4b..80ac88e8 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -7,7 +7,10 @@ var PixelFormat = require('./pixelformat') function VncConnection(conn) { this._bound = { - _readableListener: this._readableListener.bind(this) + _errorListener: this._errorListener.bind(this) + , _readableListener: this._readableListener.bind(this) + , _endListener: this._endListener.bind(this) + , _closeListener: this._closeListener.bind(this) } this._buffer = null @@ -16,12 +19,12 @@ function VncConnection(conn) { this._serverVersion = VncConnection.V3_008 this._serverSupportedSecurity = [VncConnection.SECURITY_NONE] - this._serverWidth = 800 - this._serverHeight = 600 + this._serverWidth = 720 + this._serverHeight = 1280 this._serverPixelFormat = new PixelFormat({ bitsPerPixel: 32 , depth: 24 - , bigEndianFlag: 1 + , bigEndianFlag: 0 , trueColorFlag: 1 , redMax: 255 , greenMax: 255 @@ -30,6 +33,7 @@ function VncConnection(conn) { , greenShift: 8 , blueShift: 0 }) + this._requireServerPixelFormat = true this._serverName = 'stf' this._clientVersion = null @@ -42,7 +46,10 @@ function VncConnection(conn) { this._clientCutTextLength = 0 this.conn = conn + .on('error', this._bound._errorListener) .on('readable', this._bound._readableListener) + .on('end', this._bound._endListener) + .on('close', this._bound._closeListener) this._writeServerVersion() this._read() @@ -67,6 +74,8 @@ VncConnection.CLIENT_MESSAGE_KEYEVENT = 4 VncConnection.CLIENT_MESSAGE_POINTEREVENT = 5 VncConnection.CLIENT_MESSAGE_CLIENTCUTTEXT = 6 +VncConnection.SERVER_MESSAGE_FBUPDATE = 0 + var StateReverse = Object.create(null), State = { STATE_NEED_CLIENT_VERSION: 10 , STATE_NEED_CLIENT_SECURITY: 20 @@ -82,11 +91,61 @@ var StateReverse = Object.create(null), State = { , STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE: 101 } +VncConnection.ENCODING_RAW = 0 + Object.keys(State).map(function(name) { VncConnection[name] = State[name] StateReverse[State[name]] = name }) +VncConnection.prototype.end = function() { + this.conn.end() +} + +VncConnection.prototype.writeFramebufferUpdate = function(rectangles) { + var chunk = new Buffer(4) + chunk[0] = VncConnection.SERVER_MESSAGE_FBUPDATE + chunk[1] = 0 + chunk.writeUInt16BE(rectangles.length, 2) + this._write(chunk) + + rectangles.forEach(function(rect) { + var chunk = new Buffer(12) + chunk.writeUInt16BE(rect.xPosition, 0) + chunk.writeUInt16BE(rect.yPosition, 2) + chunk.writeUInt16BE(rect.width, 4) + chunk.writeUInt16BE(rect.height, 6) + chunk.writeInt32BE(rect.encodingType, 8) + this._write(chunk) + + switch (rect.encodingType) { + case VncConnection.ENCODING_RAW: + this._write(rect.data) + break + default: + throw new Error(util.format( + 'Unsupported encoding type', rect.encodingType)) + } + }, this) +} + +VncConnection.prototype._error = function(err) { + this.emit('error', err) + this.end() +} + +VncConnection.prototype._errorListener = function(err) { + this._error(err) +} + +VncConnection.prototype._endListener = function() { + this.emit('end') +} + +VncConnection.prototype._closeListener = function() { + this.emit('close') +} + VncConnection.prototype._writeServerVersion = function() { // Yes, we could just format the string instead. Didn't feel like it. switch (this._serverVersion) { @@ -171,7 +230,10 @@ VncConnection.prototype._read = function() { switch (this._state) { case VncConnection.STATE_NEED_CLIENT_VERSION: if ((chunk = this._consume(12))) { - this._clientVersion = this._parseVersion(chunk) + if ((this._clientVersion = this._parseVersion(chunk)) === null) { + this.end() + return + } debug('client version', this._clientVersion) this._writeSupportedSecurity() this._changeState(VncConnection.STATE_NEED_CLIENT_SECURITY) @@ -179,7 +241,12 @@ VncConnection.prototype._read = function() { break case VncConnection.STATE_NEED_CLIENT_SECURITY: if ((chunk = this._consume(1))) { - this._clientSecurity = this._parseSecurity(chunk) + if ((this._clientSecurity = this._parseSecurity(chunk)) === null) { + this._writeSecurityResult( + VncConnection.SECURITYRESULT_FAIL, 'Unsupported security type') + this.end() + return + } debug('client security', this._clientSecurity) this._writeSecurityResult(VncConnection.SECURITYRESULT_OK) this._changeState(VncConnection.STATE_NEED_CLIENT_INIT) @@ -197,25 +264,33 @@ VncConnection.prototype._read = function() { if ((chunk = this._consume(1))) { switch (chunk[0]) { case VncConnection.CLIENT_MESSAGE_SETPIXELFORMAT: - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETPIXELFORMAT) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_SETPIXELFORMAT) break case VncConnection.CLIENT_MESSAGE_SETENCODINGS: - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS) break case VncConnection.CLIENT_MESSAGE_FBUPDATEREQUEST: - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST) break case VncConnection.CLIENT_MESSAGE_KEYEVENT: - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_KEYEVENT) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_KEYEVENT) break case VncConnection.CLIENT_MESSAGE_POINTEREVENT: - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_POINTEREVENT) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_POINTEREVENT) break case VncConnection.CLIENT_MESSAGE_CLIENTCUTTEXT: - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT) break default: - throw new Error(util.format('Unsupported message type %d', chunk[0])) + this._error(new Error(util.format( + 'Unsupported message type %d', chunk[0]))) + return } } break @@ -236,6 +311,12 @@ VncConnection.prototype._read = function() { }) // [16b, 19b) padding debug('client pixel format', this._clientPixelFormat) + if (this._requireServerPixelFormat && + this._clientPixelFormat.bitsPerPixel < + this._serverPixelFormat.bitsPerPixel) { + this.end() + return + } this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) } break @@ -243,7 +324,8 @@ VncConnection.prototype._read = function() { if ((chunk = this._consume(3))) { // [0b, 1b) padding this._clientEncodingCount = chunk.readUInt16BE(1, true) - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_VALUE) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_VALUE) } break case VncConnection.STATE_NEED_CLIENT_MESSAGE_SETENCODINGS_VALUE: @@ -261,11 +343,13 @@ VncConnection.prototype._read = function() { break case VncConnection.STATE_NEED_CLIENT_MESSAGE_FBUPDATEREQUEST: if ((chunk = this._consume(9))) { - // incremental = chunk[0] - // xPosition = chunk.readUInt16BE(1, true) - // yPosition = chunk.readUInt16BE(3, true) - // width = chunk.readUInt16BE(5, true) - // height = chunk.readUInt16BE(7, true) + this.emit('fbupdaterequest', { + incremental: chunk[0] + , xPosition: chunk.readUInt16BE(1, true) + , yPosition: chunk.readUInt16BE(3, true) + , width: chunk.readUInt16BE(5, true) + , height: chunk.readUInt16BE(7, true) + }) this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) } break @@ -289,7 +373,8 @@ VncConnection.prototype._read = function() { if ((chunk = this._consume(7))) { // [0b, 3b) padding this._clientCutTextLength = chunk.readUInt32BE(3) - this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE) + this._changeState( + VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE) } break case VncConnection.STATE_NEED_CLIENT_MESSAGE_CLIENTCUTTEXT_VALUE: @@ -319,7 +404,7 @@ VncConnection.prototype._parseVersion = function(chunk) { return VncConnection.V3_003 } - throw new Error('Unsupported version') + return null } VncConnection.prototype._parseSecurity = function(chunk) { @@ -328,7 +413,7 @@ VncConnection.prototype._parseSecurity = function(chunk) { case VncConnection.SECURITY_VNC: return chunk[0] default: - throw new Error('Unsupported security type') + return null } } diff --git a/lib/units/device/plugins/vnc/util/example.js b/lib/units/device/plugins/vnc/util/example.js deleted file mode 100644 index 6ed338e0..00000000 --- a/lib/units/device/plugins/vnc/util/example.js +++ /dev/null @@ -1,10 +0,0 @@ -var net = require('net') -var VncServer = require('./server') - -var nserv = net.createServer({ - allowHalfOpen: true -}) - -var vserv = new VncServer(nserv) - -nserv.listen(5910) diff --git a/lib/units/device/plugins/vnc/util/server.js b/lib/units/device/plugins/vnc/util/server.js index 2c8f6162..33b6c8e2 100644 --- a/lib/units/device/plugins/vnc/util/server.js +++ b/lib/units/device/plugins/vnc/util/server.js @@ -1,4 +1,3 @@ -var net = require('net') var util = require('util') var EventEmitter = require('eventemitter3').EventEmitter @@ -8,18 +7,44 @@ var VncConnection = require('./connection') function VncServer(server) { this._bound = { - _connectionListener: this._connectionListener.bind(this) + _listeningListener: this._listeningListener.bind(this) + , _connectionListener: this._connectionListener.bind(this) + , _closeListener: this._closeListener.bind(this) + , _errorListener: this._errorListener.bind(this) } this.server = server + .on('listening', this._bound._listeningListener) .on('connection', this._bound._connectionListener) + .on('close', this._bound._closeListener) + .on('error', this._bound._errorListener) } util.inherits(VncServer, EventEmitter) +VncServer.prototype.close = function() { + this.server.close() +} + +VncServer.prototype.listen = function() { + this.server.listen.apply(this.server, arguments) +} + +VncServer.prototype._listeningListener = function() { + this.emit('listening') +} + VncServer.prototype._connectionListener = function(conn) { debug('connection', conn.remoteAddress, conn.remotePort) - new VncConnection(conn) + this.emit('connection', new VncConnection(conn)) +} + +VncServer.prototype._closeListener = function() { + this.emit('close') +} + +VncServer.prototype._errorListener = function(err) { + this.emit('error', err) } module.exports = VncServer From 797e97e7c7353424cd9b0ceef69cde0c800139e6 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 19:34:06 +0900 Subject: [PATCH 08/23] Tell new broadcast sessions what the current state is, just in case they were added in the middle of an existing session. --- lib/units/device/plugins/screen/stream.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/units/device/plugins/screen/stream.js b/lib/units/device/plugins/screen/stream.js index bc9e5b7d..783296d8 100644 --- a/lib/units/device/plugins/screen/stream.js +++ b/lib/units/device/plugins/screen/stream.js @@ -455,6 +455,10 @@ module.exports = syrup.serial() frameProducer.stop() }) + broadcastSet.on('insert', function(id) { + broadcastSet.get(id).onStart(frameProducer) + }) + display.on('rotationChange', function(newRotation) { frameProducer.updateRotation(newRotation) }) From 692c043f6e91928b1b5deef608ee0c69472c1d21 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 19:35:53 +0900 Subject: [PATCH 09/23] Send latest screen size with every update. --- lib/units/device/plugins/vnc/index.js | 23 ++++++++++++------- .../device/plugins/vnc/util/connection.js | 3 +++ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 66581f3b..7593e962 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -90,14 +90,21 @@ module.exports = syrup.serial() } var decoded = jpeg.decode(connState.lastFrame) - conn.writeFramebufferUpdate([{ - xPosition: 0 - , yPosition: 0 - , width: connState.frameWidth - , height: connState.frameHeight - , encodingType: VncConnection.ENCODING_RAW - , data: decoded.data - }]) + conn.writeFramebufferUpdate([ + { xPosition: 0 + , yPosition: 0 + , width: decoded.width + , height: decoded.height + , encodingType: VncConnection.ENCODING_RAW + , data: decoded.data + } + , { xPosition: 0 + , yPosition: 0 + , width: decoded.width + , height: decoded.height + , encodingType: VncConnection.ENCODING_DESKTOPSIZE + } + ]) connState.updateRequests = 0 connState.sentFrameTime = connState.lastFrameTime diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 80ac88e8..eeaf24ca 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -92,6 +92,7 @@ var StateReverse = Object.create(null), State = { } VncConnection.ENCODING_RAW = 0 +VncConnection.ENCODING_DESKTOPSIZE = -223 Object.keys(State).map(function(name) { VncConnection[name] = State[name] @@ -122,6 +123,8 @@ VncConnection.prototype.writeFramebufferUpdate = function(rectangles) { case VncConnection.ENCODING_RAW: this._write(rect.data) break + case VncConnection.ENCODING_DESKTOPSIZE: + break default: throw new Error(util.format( 'Unsupported encoding type', rect.encodingType)) From bff4b7bafe144ac2dc11b866e86c07aa57e38757 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 19:36:40 +0900 Subject: [PATCH 10/23] Make sure chunk gets reset after every read loop. --- lib/units/device/plugins/vnc/util/connection.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index eeaf24ca..5041c675 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -230,6 +230,7 @@ VncConnection.prototype._read = function() { while (this._append(this.conn.read())) { do { debug('state', StateReverse[this._state]) + chunk = null switch (this._state) { case VncConnection.STATE_NEED_CLIENT_VERSION: if ((chunk = this._consume(12))) { From 5cc27a86ad2a50f7c49f85616f7413ffe150777d Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 19:38:12 +0900 Subject: [PATCH 11/23] Don't add clients to the broadcast set until they're authenticated. Note that actual, real auth is not checked yet. --- lib/units/device/plugins/vnc/index.js | 8 +++++--- lib/units/device/plugins/vnc/util/connection.js | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 7593e962..8ae2b69a 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -110,9 +110,11 @@ module.exports = syrup.serial() connState.sentFrameTime = connState.lastFrameTime } - screenStream.broadcastSet.insert(id, { - onStart: vncStartListener - , onFrame: vncFrameListener + conn.on('authenticated', function() { + screenStream.broadcastSet.insert(id, { + onStart: vncStartListener + , onFrame: vncFrameListener + }) }) conn.on('fbupdaterequest', function() { diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 5041c675..270bcc98 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -253,6 +253,7 @@ VncConnection.prototype._read = function() { } debug('client security', this._clientSecurity) this._writeSecurityResult(VncConnection.SECURITYRESULT_OK) + this.emit('authenticated') this._changeState(VncConnection.STATE_NEED_CLIENT_INIT) } break From 4b03fd81b39a3f512e8c2a915f9403bdf1716337 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 19:39:52 +0900 Subject: [PATCH 12/23] Remove unused client attributes. --- lib/units/device/plugins/vnc/util/connection.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 270bcc98..6d219239 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -38,8 +38,6 @@ function VncConnection(conn) { this._clientVersion = null this._clientShare = false - this._clientWidth = this._serverWidth - this._clientHeight = this._serverHeight this._clientPixelFormat = this._serverPixelFormat this._clientEncodingCount = 0 this._clientEncodings = [] From 02af21e17c7a307594733ffd9ba589a33d9f4864 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 19:40:40 +0900 Subject: [PATCH 13/23] Pass VNC server name in options. --- lib/units/device/plugins/vnc/index.js | 6 +++++- lib/units/device/plugins/vnc/util/connection.js | 6 ++++-- lib/units/device/plugins/vnc/util/server.js | 6 ++++-- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 8ae2b69a..2a697680 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -20,9 +20,13 @@ module.exports = syrup.serial() function createServer() { log.info('Starting VNC server on port %d', options.vncPort) + var opts = { + name: options.serial + } + var vnc = new VncServer(net.createServer({ allowHalfOpen: true - })) + }), opts) var listeningListener, errorListener return new Promise(function(resolve, reject) { diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 6d219239..8ce5382e 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -5,7 +5,9 @@ var debug = require('debug')('vnc:connection') var PixelFormat = require('./pixelformat') -function VncConnection(conn) { +function VncConnection(conn, options) { + this.options = options + this._bound = { _errorListener: this._errorListener.bind(this) , _readableListener: this._readableListener.bind(this) @@ -34,7 +36,7 @@ function VncConnection(conn) { , blueShift: 0 }) this._requireServerPixelFormat = true - this._serverName = 'stf' + this._serverName = this.options.name this._clientVersion = null this._clientShare = false diff --git a/lib/units/device/plugins/vnc/util/server.js b/lib/units/device/plugins/vnc/util/server.js index 33b6c8e2..5a8b2382 100644 --- a/lib/units/device/plugins/vnc/util/server.js +++ b/lib/units/device/plugins/vnc/util/server.js @@ -5,7 +5,9 @@ var debug = require('debug')('vnc:server') var VncConnection = require('./connection') -function VncServer(server) { +function VncServer(server, options) { + this.options = options + this._bound = { _listeningListener: this._listeningListener.bind(this) , _connectionListener: this._connectionListener.bind(this) @@ -36,7 +38,7 @@ VncServer.prototype._listeningListener = function() { VncServer.prototype._connectionListener = function(conn) { debug('connection', conn.remoteAddress, conn.remotePort) - this.emit('connection', new VncConnection(conn)) + this.emit('connection', new VncConnection(conn, this.options)) } VncServer.prototype._closeListener = function() { From 2ee767ffedddd9375c8d2e3451ee59f6e198e2a7 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 20:33:29 +0900 Subject: [PATCH 14/23] Don't send manual onStart() if frameproducer's not ready. --- lib/units/device/plugins/screen/stream.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/units/device/plugins/screen/stream.js b/lib/units/device/plugins/screen/stream.js index 783296d8..771fe28a 100644 --- a/lib/units/device/plugins/screen/stream.js +++ b/lib/units/device/plugins/screen/stream.js @@ -456,7 +456,16 @@ module.exports = syrup.serial() }) broadcastSet.on('insert', function(id) { - broadcastSet.get(id).onStart(frameProducer) + // If two clients join a session in the middle, one of them + // may not release the initial size because the projection + // doesn't necessarily change, and the producer doesn't Getting + // restarted. Therefore we have to call onStart() manually + // if the producer is already up and running. + switch (frameProducer.runningState) { + case FrameProducer.STATE_STARTED: + broadcastSet.get(id).onStart(frameProducer) + break + } }) display.on('rotationChange', function(newRotation) { From a98cc67e2c539b344a598fd99c943a8f1d4cc723 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 14 Sep 2015 20:33:59 +0900 Subject: [PATCH 15/23] Translate pointer events into touch events. --- lib/units/device/plugins/vnc/index.js | 26 +++++++- .../device/plugins/vnc/util/connection.js | 12 ++-- .../plugins/vnc/util/pointertranslator.js | 66 +++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 lib/units/device/plugins/vnc/util/pointertranslator.js diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 2a697680..9829d5c8 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -11,10 +11,12 @@ var lifecycle = require('../../../../util/lifecycle') var VncServer = require('./util/server') var VncConnection = require('./util/connection') +var PointerTranslator = require('./util/pointertranslator') module.exports = syrup.serial() .dependency(require('../screen/stream')) - .define(function(options, screenStream) { + .dependency(require('../touch')) + .define(function(options, screenStream, touch) { var log = logger.createLogger('device:plugins:vnc') function createServer() { @@ -63,6 +65,24 @@ module.exports = syrup.serial() , updateRequests: 0 } + var pointerTranslator = new PointerTranslator() + + pointerTranslator.on('touchdown', function(event) { + touch.touchDown(event) + }) + + pointerTranslator.on('touchmove', function(event) { + touch.touchMove(event) + }) + + pointerTranslator.on('touchup', function(event) { + touch.touchUp(event) + }) + + pointerTranslator.on('touchcommit', function() { + touch.touchCommit() + }) + function vncStartListener(frameProducer) { return new Promise(function(resolve/*, reject*/) { connState.frameWidth = frameProducer.banner.virtualWidth @@ -126,6 +146,10 @@ module.exports = syrup.serial() maybeSendFrame() }) + conn.on('pointer', function(event) { + pointerTranslator.push(event) + }) + conn.on('close', function() { screenStream.broadcastSet.remove(id) }) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 8ce5382e..0bce128b 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -21,8 +21,8 @@ function VncConnection(conn, options) { this._serverVersion = VncConnection.V3_008 this._serverSupportedSecurity = [VncConnection.SECURITY_NONE] - this._serverWidth = 720 - this._serverHeight = 1280 + this._serverWidth = 1080 + this._serverHeight = 1920 this._serverPixelFormat = new PixelFormat({ bitsPerPixel: 32 , depth: 24 @@ -368,9 +368,11 @@ VncConnection.prototype._read = function() { break case VncConnection.STATE_NEED_CLIENT_MESSAGE_POINTEREVENT: if ((chunk = this._consume(5))) { - // buttonMask = chunk[0] - // xPosition = chunk.readUInt16BE(1, true) - // yPosition = chunk.readUInt16BE(3, true) + this.emit('pointer', { + buttonMask: chunk[0] + , xPosition: chunk.readUInt16BE(1, true) / this._serverWidth + , yPosition: chunk.readUInt16BE(3, true) / this._serverHeight + }) this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) } break diff --git a/lib/units/device/plugins/vnc/util/pointertranslator.js b/lib/units/device/plugins/vnc/util/pointertranslator.js new file mode 100644 index 00000000..8161efa8 --- /dev/null +++ b/lib/units/device/plugins/vnc/util/pointertranslator.js @@ -0,0 +1,66 @@ +var util = require('util') + +var EventEmitter = require('eventemitter3').EventEmitter + +function PointerTranslator() { + this.previousEvent = null +} + +util.inherits(PointerTranslator, EventEmitter) + +PointerTranslator.prototype.push = function(event) { + if (event.buttonMask & 0xFE) { + // Non-primary buttons included, ignore. + return + } + + if (this.previousEvent) { + var buttonChanges = event.buttonMask ^ this.previousEvent.buttonMask + + // If the primary button changed, we have an up/down event. + if (buttonChanges & 1) { + // If it's pressed now, that's a down event. + if (event.buttonMask & 1) { + this.emit('touchdown', { + contact: 1 + , x: event.xPosition + , y: event.yPosition + }) + this.emit('touchcommit') + } + // It's not pressed, so we have an up event. + else { + this.emit('touchup', { + contact: 1 + }) + this.emit('touchcommit') + } + } + // Otherwise, if we're still holding the primary button down, + // that's a move event. + else if (event.buttonMask & 1) { + this.emit('touchmove', { + contact: 1 + , x: event.xPosition + , y: event.yPosition + }) + this.emit('touchcommit') + } + } + else { + // If it's the first event we get and the primary button's pressed, + // it's a down event. + if (event.buttonMask & 1) { + this.emit('touchdown', { + contact: 1 + , x: event.xPosition + , y: event.yPosition + }) + this.emit('touchcommit') + } + } + + this.previousEvent = event +} + +module.exports = PointerTranslator From ced0bf99a7995dc15e7995171798f586bfdc9185 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 15 Sep 2015 04:01:16 +0900 Subject: [PATCH 16/23] Use jpeg-turbo for decompressing the JPGs. It's super fast compared to what it was before. --- README.md | 3 ++- lib/units/device/plugins/vnc/index.js | 4 ++-- lib/units/device/plugins/vnc/util/connection.js | 2 +- package.json | 1 + 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 050e5460..3eb6e20f 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,7 @@ As the product has evolved from an internal tool running in our internal network * [GraphicsMagick](http://www.graphicsmagick.org/) (for resizing screenshots) * [ZeroMQ](http://zeromq.org/) libraries installed * [Protocol Buffers](https://github.com/google/protobuf) libraries installed +* [jpeg-turbo](http://libjpeg-turbo.virtualgl.org/) libraries installed (for quick VNC format conversions) * [pkg-config](http://www.freedesktop.org/wiki/Software/pkg-config/) so that Node.js can find the libraries Note that you need these dependencies even if you've installed STF directly from [NPM](https://www.npmjs.com/), because they can't be included. @@ -72,7 +73,7 @@ Note that you need these dependencies even if you've installed STF directly from On OS X, you can use [homebrew](http://brew.sh/) to install most of the dependencies: ```bash -brew install rethinkdb graphicsmagick zeromq protobuf pkg-config +brew install rethinkdb graphicsmagick zeromq protobuf jpeg-turbo pkg-config ``` On Windows you're on your own. In theory you might be able to get STF installed via [Cygwin](https://www.cygwin.com/) or similar, but we've never tried. In principle we will not provide any Windows installation support, but please do send a documentation pull request if you figure out what to do. diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 9829d5c8..8398f3d8 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -4,7 +4,7 @@ var util = require('util') var syrup = require('stf-syrup') var Promise = require('bluebird') var uuid = require('node-uuid') -var jpeg = require('jpeg-js') +var jpeg = require('jpeg-turbo') var logger = require('../../../../util/logger') var lifecycle = require('../../../../util/lifecycle') @@ -113,7 +113,7 @@ module.exports = syrup.serial() return } - var decoded = jpeg.decode(connState.lastFrame) + var decoded = jpeg.decompressSync(connState.lastFrame) conn.writeFramebufferUpdate([ { xPosition: 0 , yPosition: 0 diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 0bce128b..fe43028b 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -24,7 +24,7 @@ function VncConnection(conn, options) { this._serverWidth = 1080 this._serverHeight = 1920 this._serverPixelFormat = new PixelFormat({ - bitsPerPixel: 32 + bitsPerPixel: 24 , depth: 24 , bigEndianFlag: 0 , trueColorFlag: 1 diff --git a/package.json b/package.json index 879024d1..93a20d0a 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "http-proxy": "^1.11.2", "in-publish": "^2.0.0", "jade": "^1.9.2", + "jpeg-turbo": "^0.1.1", "jws": "^3.1.0", "ldapjs": "git+https://github.com/mcavage/node-ldapjs.git#acc1ca8f4314fd9d67561feabc8ce4c235076a5e", "lodash": "^3.10.1", From d5bfcb16d4c54d5db39bb156e22c86c6e7d42d43 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 15 Sep 2015 04:02:21 +0900 Subject: [PATCH 17/23] Update server width based on desktop size. --- lib/units/device/plugins/vnc/util/connection.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index fe43028b..7233c9b0 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -124,6 +124,8 @@ VncConnection.prototype.writeFramebufferUpdate = function(rectangles) { this._write(rect.data) break case VncConnection.ENCODING_DESKTOPSIZE: + this._serverWidth = rect.width + this._serverHeight = rect.height break default: throw new Error(util.format( From 9d20484dcb3b292a2569bf749cae99a7c29541b4 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 15 Sep 2015 04:25:21 +0900 Subject: [PATCH 18/23] Limit VNC size by default. --- lib/cli.js | 15 +++++++++++++++ lib/units/device/plugins/screen/stream.js | 6 +++--- lib/units/device/plugins/vnc/index.js | 4 ++++ lib/units/device/plugins/vnc/util/connection.js | 4 ++-- lib/util/cliutil.js | 5 +++++ 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index a10a6e44..ddd9c5b4 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -68,6 +68,10 @@ program , 'adb connect URL pattern' , String , '${publicIp}:${publicPort}') + .option('--vnc-initial-size ' + , 'initial VNC size' + , cliutil.size + , [600, 800]) .option('--mute-master' , 'whether to mute master volume when devices are being used') .option('--lock-rotation' @@ -112,6 +116,7 @@ program , '--screen-ws-url-pattern', options.screenWsUrlPattern , '--connect-url-pattern', options.connectUrlPattern , '--heartbeat-interval', options.heartbeatInterval + , '--vnc-initial-size', options.vncInitialSize.join('x') ] .concat(options.muteMaster ? ['--mute-master'] : []) .concat(options.lockRotation ? ['--lock-rotation'] : [])) @@ -146,6 +151,10 @@ program .option('--vnc-port ' , 'port allocated to vnc' , Number) + .option('--vnc-initial-size ' + , 'initial VNC size' + , cliutil.size + , [600, 800]) .option('--connect-url-pattern ' , 'adb connect URL pattern' , String @@ -221,6 +230,7 @@ program , connectUrlPattern: options.connectUrlPattern , connectPort: options.connectPort , vncPort: options.vncPort + , vncInitialSize: options.vncInitialSize , heartbeatInterval: options.heartbeatInterval , muteMaster: options.muteMaster , lockRotation: options.lockRotation @@ -954,6 +964,10 @@ program .option('--user-profile-url ' , 'URL to external user profile page' , String) + .option('--vnc-initial-size ' + , 'initial VNC size' + , cliutil.size + , [600, 800]) .option('--mute-master' , 'whether to mute master volume when devices are being used') .option('--lock-rotation' @@ -1021,6 +1035,7 @@ program , util.format('http://localhost:%d/', options.poorxyPort) , '--adb-host', options.adbHost , '--adb-port', options.adbPort + , '--vnc-initial-size', options.vncInitialSize.join('x') ] .concat(options.allowRemote ? ['--allow-remote'] : []) .concat(options.muteMaster ? ['--mute-master'] : []) diff --git a/lib/units/device/plugins/screen/stream.js b/lib/units/device/plugins/screen/stream.js index 771fe28a..f8a6f25d 100644 --- a/lib/units/device/plugins/screen/stream.js +++ b/lib/units/device/plugins/screen/stream.js @@ -25,7 +25,6 @@ module.exports = syrup.serial() .dependency(require('./options')) .define(function(options, adb, minicap, display, screenOptions) { var log = logger.createLogger('device:plugins:screen:stream') - var plugin = Object.create(null) function FrameProducer(config) { EventEmitter.call(this) @@ -443,9 +442,9 @@ module.exports = syrup.serial() return createServer() .then(function(wss) { - var broadcastSet = plugin.broadcastSet = new BroadcastSet() var frameProducer = new FrameProducer( new FrameConfig(display.properties, display.properties)) + var broadcastSet = frameProducer.broadcastSet = new BroadcastSet() broadcastSet.on('nonempty', function() { frameProducer.start() @@ -589,6 +588,7 @@ module.exports = syrup.serial() lifecycle.observe(function() { frameProducer.stop() }) + + return frameProducer }) - .return(plugin) }) diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 8398f3d8..67956b50 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -24,6 +24,8 @@ module.exports = syrup.serial() var opts = { name: options.serial + , width: options.vncInitialSize[0] + , height: options.vncInitialSize[1] } var vnc = new VncServer(net.createServer({ @@ -135,6 +137,8 @@ module.exports = syrup.serial() } conn.on('authenticated', function() { + screenStream.updateProjection( + options.vncInitialSize[0], options.vncInitialSize[1]) screenStream.broadcastSet.insert(id, { onStart: vncStartListener , onFrame: vncFrameListener diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 7233c9b0..68f3c99f 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -21,8 +21,8 @@ function VncConnection(conn, options) { this._serverVersion = VncConnection.V3_008 this._serverSupportedSecurity = [VncConnection.SECURITY_NONE] - this._serverWidth = 1080 - this._serverHeight = 1920 + this._serverWidth = this.options.width + this._serverHeight = this.options.height this._serverPixelFormat = new PixelFormat({ bitsPerPixel: 24 , depth: 24 diff --git a/lib/util/cliutil.js b/lib/util/cliutil.js index 12260802..818aac01 100644 --- a/lib/util/cliutil.js +++ b/lib/util/cliutil.js @@ -2,6 +2,11 @@ module.exports.list = function(val) { return val.split(/\s*,\s*/g).filter(Boolean) } +module.exports.size = function(val) { + var match = /^(\d+)x(\d+)$/.exec(val) + return match ? [+match[1], +match[2]] : undefined +} + module.exports.allUnknownArgs = function(args) { return [].slice.call(args, 0, -1).filter(Boolean) } From 977b8c198e83be1bfecfa41e99cc28c1050f842a Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 21 Sep 2015 16:28:05 +0900 Subject: [PATCH 19/23] Support variable bit depths. Fix color issues. --- lib/units/device/plugins/vnc/index.js | 37 ++++++++++++++++++- .../device/plugins/vnc/util/connection.js | 12 ++---- package.json | 2 +- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/lib/units/device/plugins/vnc/index.js b/lib/units/device/plugins/vnc/index.js index 67956b50..0a896993 100644 --- a/lib/units/device/plugins/vnc/index.js +++ b/lib/units/device/plugins/vnc/index.js @@ -1,5 +1,6 @@ var net = require('net') var util = require('util') +var os = require('os') var syrup = require('stf-syrup') var Promise = require('bluebird') @@ -65,6 +66,9 @@ module.exports = syrup.serial() , frameHeight: 0 , sentFrameTime: null , updateRequests: 0 + , frameConfig: { + format: jpeg.FORMAT_RGB + } } var pointerTranslator = new PointerTranslator() @@ -115,7 +119,9 @@ module.exports = syrup.serial() return } - var decoded = jpeg.decompressSync(connState.lastFrame) + var decoded = jpeg.decompressSync( + connState.lastFrame, connState.frameConfig) + conn.writeFramebufferUpdate([ { xPosition: 0 , yPosition: 0 @@ -150,6 +156,35 @@ module.exports = syrup.serial() maybeSendFrame() }) + conn.on('formatchange', function(format) { + var same = os.endianness() == 'BE' == format.bigEndianFlag + switch (format.bitsPerPixel) { + case 8: + connState.frameConfig = { + format: jpeg.FORMAT_GRAY + } + break + case 24: + connState.frameConfig = { + format: ((format.redShift > format.blueShift) === same) + ? jpeg.FORMAT_BGR + : jpeg.FORMAT_RGB + } + break + case 32: + connState.frameConfig = { + format: ((format.redShift > format.blueShift) === same) + ? (format.blueShift === 0 + ? jpeg.FORMAT_BGRX + : jpeg.FORMAT_XBGR) + : (format.redShift === 0 + ? jpeg.FORMAT_RGBX + : jpeg.FORMAT_XRGB) + } + break + } + }) + conn.on('pointer', function(event) { pointerTranslator.push(event) }) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index 68f3c99f..c80870f0 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -1,4 +1,5 @@ var util = require('util') +var os = require('os') var EventEmitter = require('eventemitter3').EventEmitter var debug = require('debug')('vnc:connection') @@ -26,7 +27,7 @@ function VncConnection(conn, options) { this._serverPixelFormat = new PixelFormat({ bitsPerPixel: 24 , depth: 24 - , bigEndianFlag: 0 + , bigEndianFlag: os.endianness() == 'BE' ? 1 : 0 , trueColorFlag: 1 , redMax: 255 , greenMax: 255 @@ -35,7 +36,6 @@ function VncConnection(conn, options) { , greenShift: 8 , blueShift: 0 }) - this._requireServerPixelFormat = true this._serverName = this.options.name this._clientVersion = null @@ -202,6 +202,7 @@ VncConnection.prototype._writeSecurityResult = function(result, reason) { } VncConnection.prototype._writeServerInit = function() { + debug('server pixel format', this._serverPixelFormat) var chunk = new Buffer(2 + 2 + 16 + 4 + this._serverName.length) chunk.writeUInt16BE(this._serverWidth, 0) chunk.writeUInt16BE(this._serverHeight, 2) @@ -318,12 +319,7 @@ VncConnection.prototype._read = function() { }) // [16b, 19b) padding debug('client pixel format', this._clientPixelFormat) - if (this._requireServerPixelFormat && - this._clientPixelFormat.bitsPerPixel < - this._serverPixelFormat.bitsPerPixel) { - this.end() - return - } + this.emit('formatchange', this._clientPixelFormat) this._changeState(VncConnection.STATE_NEED_CLIENT_MESSAGE) } break diff --git a/package.json b/package.json index 93a20d0a..acb01d95 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "http-proxy": "^1.11.2", "in-publish": "^2.0.0", "jade": "^1.9.2", - "jpeg-turbo": "^0.1.1", + "jpeg-turbo": "^0.2.1", "jws": "^3.1.0", "ldapjs": "git+https://github.com/mcavage/node-ldapjs.git#acc1ca8f4314fd9d67561feabc8ce4c235076a5e", "lodash": "^3.10.1", From 1f32c21c78001f9e0bc70f6654c6c2598ae8bea7 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 21 Sep 2015 16:38:03 +0900 Subject: [PATCH 20/23] Use 32 bits per pixel by default, since 24 bpp doesn't seem to be a supported option in the RFB 3.8 spec. --- lib/units/device/plugins/vnc/util/connection.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/units/device/plugins/vnc/util/connection.js b/lib/units/device/plugins/vnc/util/connection.js index c80870f0..7d7868ae 100644 --- a/lib/units/device/plugins/vnc/util/connection.js +++ b/lib/units/device/plugins/vnc/util/connection.js @@ -25,7 +25,7 @@ function VncConnection(conn, options) { this._serverWidth = this.options.width this._serverHeight = this.options.height this._serverPixelFormat = new PixelFormat({ - bitsPerPixel: 24 + bitsPerPixel: 32 , depth: 24 , bigEndianFlag: os.endianness() == 'BE' ? 1 : 0 , trueColorFlag: 1 From bb3e8fb3d5f25222203d7db5b8db1f66e22d7f07 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Tue, 22 Sep 2015 11:54:57 +0900 Subject: [PATCH 21/23] Update jpeg-turbo, removes libjpeg-turbo from requirements but adds yasm. --- .travis.yml | 1 + README.md | 6 +++--- package.json | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8e633bb4..de2cc8d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ addons: - libprotobuf-dev - graphicsmagick - rethinkdb + - yasm script: - gulp build before_script: diff --git a/README.md b/README.md index 3eb6e20f..93034a49 100644 --- a/README.md +++ b/README.md @@ -65,15 +65,15 @@ As the product has evolved from an internal tool running in our internal network * [GraphicsMagick](http://www.graphicsmagick.org/) (for resizing screenshots) * [ZeroMQ](http://zeromq.org/) libraries installed * [Protocol Buffers](https://github.com/google/protobuf) libraries installed -* [jpeg-turbo](http://libjpeg-turbo.virtualgl.org/) libraries installed (for quick VNC format conversions) +* [yasm](http://yasm.tortall.net/) installed (for compiling embedded [libjpeg-turbo](https://github.com/sorccu/node-jpeg-turbo)) * [pkg-config](http://www.freedesktop.org/wiki/Software/pkg-config/) so that Node.js can find the libraries -Note that you need these dependencies even if you've installed STF directly from [NPM](https://www.npmjs.com/), because they can't be included. +Note that you need these dependencies even if you've installed STF directly from [NPM](https://www.npmjs.com/), because they can't be included in the package. On OS X, you can use [homebrew](http://brew.sh/) to install most of the dependencies: ```bash -brew install rethinkdb graphicsmagick zeromq protobuf jpeg-turbo pkg-config +brew install rethinkdb graphicsmagick zeromq protobuf yasm pkg-config ``` On Windows you're on your own. In theory you might be able to get STF installed via [Cygwin](https://www.cygwin.com/) or similar, but we've never tried. In principle we will not provide any Windows installation support, but please do send a documentation pull request if you figure out what to do. diff --git a/package.json b/package.json index acb01d95..20da8fee 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "http-proxy": "^1.11.2", "in-publish": "^2.0.0", "jade": "^1.9.2", - "jpeg-turbo": "^0.2.1", + "jpeg-turbo": "^0.3.0", "jws": "^3.1.0", "ldapjs": "git+https://github.com/mcavage/node-ldapjs.git#acc1ca8f4314fd9d67561feabc8ce4c235076a5e", "lodash": "^3.10.1", From 7a716a6da5d5c80a222c39340c81869dd7efc1f4 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 28 Sep 2015 14:59:07 +0900 Subject: [PATCH 22/23] Update base to v1.0.4. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a698121b..03bf8af9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM openstf/base:v1.0.2 +FROM openstf/base:v1.0.4 # Sneak the stf executable into $PATH. ENV PATH /app/bin:$PATH From 2aaa26ce4e05d6834e09f744a4a6faa99af35a12 Mon Sep 17 00:00:00 2001 From: Simo Kinnunen Date: Mon, 28 Sep 2015 15:01:02 +0900 Subject: [PATCH 23/23] Mention experimental VNC support. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 93034a49..f73a48c1 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ It is currently being used at [CyberAgent](https://www.cyberagent.co.jp/en/) to * Run any `adb` command locally, including shell access * [Android Studio](http://developer.android.com/tools/studio/index.html) and other IDE support, debug your app while watching the device screen on your browser * Supports [Chrome remote debug tools](https://developer.chrome.com/devtools/docs/remote-debugging) + - Experimental VNC support (work in progress) * Manage your device inventory - See which devices are connected, offline/unavailable (indicating a weak USB connection), unauthorized or unplugged - See who's using a device