Too much overlap in storage plugin URLs for meaningful loadbalancing and config simplicity. Make poorxy emulate the real situation and change related URLs.

This commit is contained in:
Simo Kinnunen
2015-01-07 14:04:36 +09:00
parent 7c16c40dae
commit 8b9e5d58c9
8 changed files with 70 additions and 57 deletions

View File

@@ -15,34 +15,35 @@ module.exports = syrup.serial()
plugin.store = function(type, stream, meta) {
var resolver = Promise.defer()
var req = request.post({
url: url.resolve(options.storageUrl, util.format('s/api/v1/%s', type))
var args = {
url: url.resolve(options.storageUrl, util.format('s/upload/%s', type))
}
var req = request.post(args, function(err, res, body) {
if (err) {
log.error('Upload to "%s" failed', args.url, err.stack)
resolver.reject(err)
}
, function(err, res, body) {
if (err) {
log.error('Upload failed', err.stack)
else if (res.statusCode !== 201) {
log.error('Upload to "%s" failed: HTTP %d', args.url, res.statusCode)
resolver.reject(new Error(util.format(
'Upload to "%s" failed: HTTP %d'
, args.url
, res.statusCode
)))
}
else {
try {
var result = JSON.parse(body)
log.info('Uploaded to "%s"', result.resources.file.href)
resolver.resolve(result.resources.file)
}
catch (err) {
log.error('Invalid JSON in response', err.stack, body)
resolver.reject(err)
}
else if (res.statusCode !== 201) {
log.error('Upload failed: HTTP %d', res.statusCode)
resolver.reject(new Error(util.format(
'Upload failed: HTTP %d'
, res.statusCode
)))
}
else {
try {
var result = JSON.parse(body)
log.info('Uploaded to %s', result.resources.file.href)
resolver.resolve(result.resources.file)
}
catch (err) {
log.error('Invalid JSON in response', err.stack, body)
resolver.reject(err)
}
}
}
)
})
req.form()
.append('file', stream, meta)

View File

@@ -27,7 +27,7 @@ module.exports = function(options) {
})
})
;['/s/api/v1/image/*'].forEach(function(route) {
;['/s/image/*'].forEach(function(route) {
app.all(route, function(req, res) {
proxy.web(req, res, {
target: options.storagePluginImageUrl
@@ -35,7 +35,7 @@ module.exports = function(options) {
})
})
;['/s/api/v1/apk/*'].forEach(function(route) {
;['/s/apk/*'].forEach(function(route) {
app.all(route, function(req, res) {
proxy.web(req, res, {
target: options.storagePluginApkUrl
@@ -43,7 +43,7 @@ module.exports = function(options) {
})
})
;['/s/api/v1/*'].forEach(function(route) {
;['/s/*'].forEach(function(route) {
app.all(route, function(req, res) {
proxy.web(req, res, {
target: options.storageUrl

View File

@@ -1,8 +1,9 @@
var http = require('http')
var url = require('url')
var util = require('util')
var express = require('express')
var httpProxy = require('http-proxy')
var request = require('request')
var logger = require('../../../../util/logger')
var download = require('../../../../util/download')
@@ -12,18 +13,18 @@ module.exports = function(options) {
var log = logger.createLogger('storage:plugins:apk')
, app = express()
, server = http.createServer(app)
, proxy = httpProxy.createProxyServer()
proxy.on('error', function(err) {
log.error('Proxy had an error', err.stack)
})
app.set('strict routing', true)
app.set('case sensitive routing', true)
app.set('trust proxy', true)
app.get('/s/api/v1/apk/:id/*/manifest', function(req, res) {
download(url.resolve(options.storageUrl, req.url), {
app.get('/s/apk/:id/:name/manifest', function(req, res) {
var orig = util.format(
'/s/blob/%s/%s'
, req.params.id
, req.params.name
)
download(url.resolve(options.storageUrl, orig), {
dir: options.cacheDir
})
.then(manifest)
@@ -43,10 +44,13 @@ module.exports = function(options) {
})
})
app.get('/s/api/v1/apk/:id/*', function(req, res) {
proxy.web(req, res, {
target: options.storageUrl
})
app.get('/s/apk/:id/:name', function(req, res) {
request(url.resolve(options.storageUrl, util.format(
'/s/blob/%s/%s'
, req.params.id
, req.params.name
)))
.pipe(res)
})
server.listen(options.port)

View File

@@ -1,4 +1,5 @@
var http = require('http')
var util = require('util')
var express = require('express')
@@ -20,9 +21,14 @@ module.exports = function(options) {
app.set('trust proxy', true)
app.get(
'/s/api/v1/image/:id/*/transform'
'/s/image/:id/:name'
, requtil.limit(options.concurrency, function(req, res) {
return get(req.url, options)
var orig = util.format(
'/s/blob/%s/%s'
, req.params.id
, req.params.name
)
return get(orig, options)
.then(function(stream) {
return transform(stream, {
crop: parseCrop(req.query.crop)

View File

@@ -30,7 +30,7 @@ module.exports = function(options) {
log.info('Cleaning up inactive resource "%s"', id)
})
app.post('/s/api/v1/:type/download', function(req, res) {
app.post('/s/download/:plugin', function(req, res) {
requtil.validate(req, function() {
req.checkBody('url').notEmpty()
})
@@ -46,17 +46,18 @@ module.exports = function(options) {
}
})
.then(function(file) {
var plugin = req.params.plugin
res.status(201)
.json({
success: true
, resource: {
date: new Date()
, type: req.params.type
, plugin: plugin
, id: file.id
, name: file.name
, href: util.format(
'/s/api/v1/%s/%s%s'
, req.params.type
'/s/%s/%s%s'
, plugin
, file.id
, file.name
? util.format('/%s', path.basename(file.name))
@@ -83,7 +84,7 @@ module.exports = function(options) {
})
})
app.post('/s/api/v1/:type', function(req, res) {
app.post('/s/upload/:plugin', function(req, res) {
var form = new formidable.IncomingForm()
Promise.promisify(form.parse, form)(req)
.spread(function(fields, files) {
@@ -104,14 +105,15 @@ module.exports = function(options) {
, resources: (function() {
var mapped = Object.create(null)
storedFiles.forEach(function(file) {
var plugin = req.params.plugin
mapped[file.field] = {
date: new Date()
, type: req.params.type
, plugin: plugin
, id: file.id
, name: file.name
, href: util.format(
'/s/api/v1/%s/%s%s'
, req.params.type
'/s/%s/%s%s'
, plugin
, file.id
, file.name
? util.format('/%s', path.basename(file.name))
@@ -133,7 +135,7 @@ module.exports = function(options) {
})
})
app.get('/s/api/v1/:type/:id/*', function(req, res) {
app.get('/s/blob/:id/:name', function(req, res) {
var file = storage.retrieve(req.params.id)
if (file) {
res.set('Content-Type', file.type)