diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..50f65cc9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +. +assets/ +assets_src/ +node_modules/ \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 00000000..ab2e9765 --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# quakejs + +## Quickstart + + git clone --recursive https://github.com/inolen/quakejs.git + cd quakejs + npm install + +## Assets + +## license + +MIT \ No newline at end of file diff --git a/bin/content.js b/bin/content.js new file mode 100644 index 00000000..a080b4fd --- /dev/null +++ b/bin/content.js @@ -0,0 +1,187 @@ +var _ = require('underscore'); +var async = require('async'); +var crypto = require('crypto'); +var express = require('express'); +var fs = require('fs'); +var http = require('http'); +var opt = require('optimist'); +var path = require('path'); +// var Throttle = require('throttle'); + +var argv = require('optimist') + .options({ + 'root': { + 'description': 'Root assets path', + 'demand': true + }, + 'port': { + 'description': 'Server port', + 'default': 9000 + } + }) + .argv; + +if (argv.h || argv.help) { + opt.showHelp(); + return; +} + +var compressedAssets = [ '.pk3' ]; +var currentManifest; + +function checksum(filename, callback) { + var sum = crypto.createHash('md5'); + var s = fs.ReadStream(filename); + s.on('error', function (err) { + callback(err); + }); + s.on('data', function (data) { + sum.update(data); + }); + s.on('end', function () { + callback(null, sum.digest('hex')); + }); +} + +// // debug throttled sendfile +// function sendfile2(file, res) { +// var stat = fs.statSync(file); + +// res.statusCode = 200; +// if (!res.getHeader('Content-Type')) { +// if (path.extname(file) === '.js') { +// res.setHeader('Content-Type', 'application/javascript'); +// } else { +// res.setHeader('Content-Type', 'application/octet-stream'); +// } +// } +// res.setHeader('Content-Length', stat.size); + +// var rs = fs.createReadStream(file); +// rs.pipe(new Throttle(1024 * 1024 * 5)).pipe(res); +// } + +function getMods(callback) { + fs.readdir(argv.root, function(err, files) { + if (err) return callback(err); + + async.filter(files, function (file, cb) { + var absolute = path.join(argv.root, file); + fs.stat(absolute, function (err, stats) { + if (err) return callback(err); + + return cb(stats.isDirectory()); + }); + }, function (results) { + callback(null, results); + }); + }); +} + +function getModFiles(mod, callback) { + var gamePath = path.join(argv.root, mod); + var valid = ['.pk3']; + + fs.readdir(gamePath, function(err, files) { + if (err) return callback(err); + + async.filter(files, function (file, cb) { + var ext = path.extname(file); + cb(valid.indexOf(ext) !== -1); + }, function (files) { + // Convert files to absolute paths. + files = files.map(function (file) { return path.join(gamePath, file); }); + + callback(null, files); + }); + }); +} + +function generateManifest(callback) { + console.log('generating manifest..'); + + getMods(function (err, mods) { + if (err) return callback(err); + + async.concat(mods, getModFiles, function (err, files) { + if (err) return callback(err); + + async.map(files, function (file, cb) { + fs.stat(file, function (err, stat) { + if (err) return cb(err); + + checksum(file, function (err, checksum) { + if (err) return cb(err); + + cb(null, { + name: path.relative(argv.root, file), + size: stat.size, + checksum: checksum + }); + }); + }); + }, function (err, entries) { + if (err) return callback(err); + console.log('done generating manifest, ' + entries.length + ' entries'); + callback(err, entries); + }); + }); + }); +} + +function handleManifest(req, res, next) { + res.json(currentManifest); +} + +function handleStaticAsset(req, res, next) { + var relativePath = req.params[0]; + var absolutePath = path.join(argv.root, relativePath); + var checksum = req.query.checksum; + + // Make sure they're requesting a valid asset, else return a 400. + var valid = currentManifest.some(function (entry) { + return entry.name === relativePath && entry.checksum === checksum; + }); + + if (!valid) { + res.status(400).end(); + return; + } + + console.log('serving ' + relativePath + ' ' + checksum); + + res.sendfile(absolutePath, function (err) { + if (err) return next(err); + }); + // sendfile2(absolutePath, res); +} + +(function main() { + // Setup the express app. + var app = express(); + app.use(function (req, res, next) { + res.setHeader('Access-Control-Allow-Origin', '*'); + next(); + }); + app.use(express.compress({ + filter: function(req, res) { + var ext = path.extname(req.url); + return (/json|text|javascript/).test(res.getHeader('Content-Type')) || + compressedAssets.indexOf(ext) !== -1; + } + })); + app.get('/assets/manifest.json', handleManifest); + app.get(/^\/assets\/(.+\.pk3)$/, handleStaticAsset); + + // Startup the HTTP server. + var server = http.createServer(app); + server.listen(argv.port, function () { + console.log('content server is now listening on port', server.address().address, server.address().port); + }); + + // Generate an initial manifest. + generateManifest(function (err, manifest) { + if (err) throw err; + currentManifest = manifest; + }); +})(); \ No newline at end of file diff --git a/bin/master.js b/bin/master.js new file mode 100644 index 00000000..1c0a6928 --- /dev/null +++ b/bin/master.js @@ -0,0 +1,362 @@ +var _ = require('underscore'); +var ansi = require('ansi'); +var http = require('http'); +var opt = require('optimist'); +var url = require('url'); +var WebSocketClient = require('ws'); +var WebSocketServer = require('ws').Server; +var cursor = ansi(process.stdout); + +var argv = require('optimist') + .describe('config', 'Location of the configuration file').default('config', './master.json') + .argv; + +if (argv.h || argv.help) { + opt.showHelp(); + return; +} + +var clients = []; +var servers = {}; +var pruneInterval = 350 * 1000; + +function formatOOB(data) { + var str = '\xff\xff\xff\xff' + data + '\x00'; + + var buffer = new ArrayBuffer(str.length); + var view = new Uint8Array(buffer); + + for (var i = 0; i < str.length; i++) { + view[i] = str.charCodeAt(i); + } + + return buffer; +} + +function stripOOB(buffer) { + var view = new DataView(buffer); + + if (view.getInt32(0) !== -1) { + return null; + } + + var str = ''; + for (var i = 4 /* ignore leading -1 */; i < buffer.byteLength - 1 /* ignore trailing \0 */; i++) { + var c = String.fromCharCode(view.getUint8(i)); + str += c; + } + + return str; +} + +function parseInfoString(str) { + var data = {}; + + var split = str.split('\\'); + // throw when split.length isn't even? + + for (var i = 0; i < split.length - 1; i += 2) { + var key = split[i]; + var value = split[i+1]; + data[key] = value; + } +} + +/********************************************************** + * + * messages + * + **********************************************************/ +var CHALLENGE_MIN_LENGTH = 9; +var CHALLENGE_MAX_LENGTH = 12; +var GAMENAME_LENGTH = 64; +var GAMETYPE_LENGTH = 32; + +function handleHeartbeat(conn, data) { + cursor + .brightGreen().write(conn.addr + ':' + conn.port).reset() + .write(' ---> ') + .magenta().write('heartbeat').reset() + .write('\n'); + + sendGetInfo(conn); +} + +function handleInfoResponse(conn, data) { + cursor + .brightGreen().write(conn.addr + ':' + conn.port).reset() + .write(' ---> ') + .magenta().write('infoResponse').reset() + .write('\n'); + + var info = parseInfoString(data); + + // TODO validate data + + updateServer(conn.addr, conn.port); +} + +function buildChallenge() { + var challenge = ''; + var length = CHALLENGE_MIN_LENGTH - 1 + + parseInt(Math.random() * (CHALLENGE_MAX_LENGTH - CHALLENGE_MIN_LENGTH + 1), 10); + + for (var i = 0; i < length; i++) { + var c; + do { + c = 33 + parseInt(Math.random() * (126 - 33 + 1), 10); // -> c = 33..126 + } while (c == '\\' || c == ';' || c == '"' || c == '%' || c == '/'); + + challenge += String.fromCharCode(c); + } + + return challenge; +} + +function sendGetInfo(conn) { + var challenge = buildChallenge(); + + cursor + .brightGreen().write(conn.addr + ':' + conn.port).reset() + .write(' <--- ') + .magenta().write('getinfo with challenge \"' + challenge + '\"').reset() + .write('\n'); + + var buffer = formatOOB('getinfo ' + challenge); + conn.socket.send(buffer, { binary: true }); +} + +function sendGetServersResponse(conn, servers) { + var msg = 'getserversResponse'; + for (var id in servers) { + if (!servers.hasOwnProperty(id)) { + continue; + } + var server = servers[id]; + var octets = server.addr.split('.').map(function (n) { + return parseInt(n, 10); + }); + msg += '\\'; + msg += String.fromCharCode(octets[0] & 0xff); + msg += String.fromCharCode(octets[1] & 0xff); + msg += String.fromCharCode(octets[2] & 0xff) + msg += String.fromCharCode(octets[3] & 0xff); + msg += String.fromCharCode((server.port & 0xff00) >> 8); + msg += String.fromCharCode(server.port & 0xff); + } + + cursor + .brightGreen().write(conn.addr + ':' + conn.port).reset() + .write(' <--- ') + .magenta().write('getserversResponse with ' + Object.keys(servers).length + ' server(s)').reset() + .write('\n'); + + var buffer = formatOOB(msg); + conn.socket.send(buffer, { binary: true }); +} + +/********************************************************** + * + * servers + * + **********************************************************/ +function serverid(addr, port) { + return addr + ':' + port; +} + +function updateServer(addr, port) { + var id = serverid(addr, port); + var server = servers[id]; + if (!server) { + server = servers[id] = { addr: addr, port: port }; + } + server.lastUpdate = Date.now(); + + // Send partial update to all clients. + for (var i = 0; i < clients.length; i++) { + sendGetServersResponse(clients[i], { id: server }); + } +} + +function removeServer(id) { + var server = servers[id]; + + delete servers[id]; + + cursor + .brightGreen().write(server.addr + ':' + server.port).reset() + .write(' timed out, ' + Object.keys(servers).length + ' server(s) currently registered\n'); +} + +function pruneServers() { + var now = Date.now(); + + for (var id in servers) { + if (!servers.hasOwnProperty(id)) { + continue; + } + + var server = servers[id]; + var delta = now - server.lastUpdate; + + if (delta > pruneInterval) { + removeServer(id); + } + } +} + +/********************************************************** + * + * clients + * + **********************************************************/ +function handleSubscribe(conn) { + addClient(conn); + + // Send all servers upon subscribing. + sendGetServersResponse(conn, servers); +} + +function addClient(conn) { + var idx = clients.indexOf(conn); + + if (idx !== -1) { + return; // already subscribed + } + + cursor + .brightGreen().write(conn.addr + ':' + conn.port).reset() + .write(' ---> ') + .magenta().write('subscribe').reset() + .write('\n'); + + clients.push(conn); +} + +function removeClient(conn) { + var idx = clients.indexOf(conn); + if (idx === -1) { + return; // conn may have belonged to a server + } + + var conn = clients[idx]; + + cursor + .brightGreen().write(conn.addr + ':' + conn.port).reset() + .write(' ---> ') + .magenta().write('unsubscribe').reset() + .write('\n'); + + clients.splice(idx, 1); +} + +/********************************************************** + * + * main + * + **********************************************************/ +function loadConfig() { + var config = { + port: 45735 + }; + + try { + cursor.write('loading config file from ' + argv.config + '.. '); + var data = require(argv.config); + _.extend(config, data); + cursor.write('ok\n'); + } catch (e) { + cursor.write('error\n'); + } + + return config; +} + +function getRemoteAddress(ws) { + // By default, check the underlying socket's remote address. + var address = ws._socket.remoteAddress; + + // If this is an x-forwarded-for header (meaning the request + // has been proxied), use it. + if (ws.upgradeReq.headers['x-forwarded-for']) { + address = ws.upgradeReq.headers['x-forwarded-for']; + } + + return address; +} + +function getRemotePort(ws) { + var port = ws._socket.remotePort; + + if (ws.upgradeReq.headers['x-forwarded-port']) { + port = ws.upgradeReq.headers['x-forwarded-port']; + } + + return port; +} + +function connection(ws) { + this.socket = ws; + this.addr = getRemoteAddress(ws); + this.port = getRemotePort(ws); +} + +(function main() { + var config = loadConfig(); + + var server = http.createServer(); + + var wss = new WebSocketServer({ + server: server + }); + + wss.on('connection', function (ws) { + var conn = new connection(ws); + var first = true; + + ws.on('message', function (buffer, flags) { + if (!flags.binary) { + return; + } + + buffer = (new Uint8Array(buffer)).buffer; // node Buffer to ArrayBuffer + + // check to see if this is emscripten's port identifier message + if (first && + buffer.byteLength === 10 && + buffer[0] === 255 && buffer[1] === 255 && buffer[2] === 255 && buffer[3] === 255 && + buffer[4] === 'p'.charCodeAt(0) && buffer[5] === 'o'.charCodeAt(0) && buffer[6] === 'r'.charCodeAt(0) && buffer[7] === 't'.charCodeAt(0)) { + conn.port = ((buffer[8] << 8) | buffer[9]); + } + first = false; + + var msg = stripOOB(buffer); + if (!msg) { + removeClient(conn); + return; + } + + if (msg.indexOf('heartbeat ') === 0) { + handleHeartbeat(conn, msg.substr(10)); + } else if (msg.indexOf('infoResponse\n') === 0) { + handleInfoResponse(conn, msg.substr(13)); + } else if (msg.indexOf('subscribe') === 0) { + handleSubscribe(conn); + } + }); + + ws.on('error', function (err) { + removeClient(conn); + }); + + ws.on('close', function () { + removeClient(conn); + }); + }); + + server.listen(config.port, function() { + console.log('master server is listening on port ' + server.address().port); + }); + + setInterval(pruneServers, pruneInterval); +})(); \ No newline at end of file diff --git a/bin/repak.js b/bin/repak.js new file mode 100644 index 00000000..30e46177 --- /dev/null +++ b/bin/repak.js @@ -0,0 +1,113 @@ +var fs = require('fs'); +var path = require('path'); +var repak = require('quakejs-repak'); +var sh = require('execSync'); +var spawn = require('child_process').spawn; +var temp = require('temp'); + +var src = process.argv[2]; +var dest = process.argv[3]; + +function filterHasBuffer(asset) { + return asset.buffer; +} + +function transform(asset) { + if (!asset.buffer || asset.name.indexOf('.wav') === -1) { + return asset; + } + + var tempsrc = temp.openSync('repak'); + var tempdest = temp.openSync('repak'); + + // write out the input + fs.writeSync(tempsrc.fd, asset.buffer, 0, asset.buffer.length, 0); + fs.closeSync(tempsrc.fd); + + // do the transform + var result = sh.exec('opusenc ' + tempsrc.path + ' ' + tempdest.path); + if (result.code) { + console.log('.. failed to opus encode ' + asset.name); + asset.buffer = null; + return asset; + } + + // read in the output + var stat = fs.fstatSync(tempdest.fd); + var buffer = new Buffer(stat.size); + fs.readSync(tempdest.fd, buffer, 0, stat.size, 0); + fs.closeSync(tempdest.fd); + + // update the asset + asset.name = asset.name.replace('.wav', '.opus'); + asset.buffer = buffer; + + return asset; +} + +repak(src, dest, { + filter: function (file) { + file = file.toLowerCase(); + var isBlacklistedMap = [ + 'maps/pro-q3tourney2.aas', 'maps/pro-q3tourney2.bsp', + 'maps/pro-q3tourney4.aas', 'maps/pro-q3tourney4.bsp', + 'maps/q3dm1.aas', 'maps/q3dm1.bsp', + 'maps/q3dm7.aas', 'maps/q3dm7.bsp', + 'maps/q3dm9.aas', 'maps/q3dm9.bsp', + 'maps/q3dm17.aas', 'maps/q3dm17.bsp', + 'maps/q3tourney2.aas', 'maps/q3tourney2.bsp', + 'maps/q3tourney6_ctf.aas', 'maps/q3tourney6_ctf.bsp'].indexOf(file) !== -1; + var isDemo = /\.dm[_]{0, 1}\d+/.test(file); + var isROQ = file.indexOf('.roq') !== -1; + var isLODMD3 = /_[123]{1}\.md3/.test(file); + return !isBlacklistedMap && !isDemo && !isLODMD3 && !isROQ; + }, + group: function (graph) { + var paks = {}; + var maps = graph.maps(); + + var common = graph.find(null, function (asset) { + // maps aren't referenced by anything + if (asset.type === graph.ASSET.MAP || + asset.type === graph.ASSET.AAS) { + return false; + } + return asset.ref <= 0 || asset.ref >= 3; + }).map(transform).filter(filterHasBuffer); + + // split up common paks every ~50mb + var num = 0; + var total = 0; + var assets = []; + var maxBytes = 50 * 1024 * 1024; + for (var i = 0; i < common.length; i++) { + var asset = common[i]; + + assets.push(asset); + total += asset.buffer.length; + + if (total >= maxBytes || i === common.length-1) { + paks['pak' + num + '.pk3'] = assets; + num++; + total = 0; + assets = []; + } + } + + // generate a pak for each map + for (var i = 0; i < maps.length; i++) { + var map = maps[i]; + var filename = path.basename(map).replace('.bsp', '.pk3'); + paks[filename] = graph.find(map, function (asset) { + if (asset.type === graph.ASSET.MAP) { + return true; + } + return asset.ref > 0 && asset.ref < 3; + }).map(transform).filter(filterHasBuffer); + } + + return paks; + } +}, function () { + console.log('done'); +}); \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 00000000..d41c4136 --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "quakejs", + "version": "0.0.1", + "bin": { + "quakejs-content": "bin/content.js", + "quakejs-master": "bin/master.js", + "quakejs-repak": "bin/repak.js" + }, + "repository": { + "type": "git", + "url": "git://github.com/inolen/quakejs.git" + }, + "author": "Anthony Pesch", + "license": "MIT", + "readmeFilename": "README.md", + "dependencies": { + "archiver": "~0.4.6", + "async": "~0.2.9", + "execSync": "~1.0.1-pre", + "quakejs-files": "0.0.1", + "temp": "~0.5.1", + "unzip": "git://github.com/nearinfinity/node-unzip.git#max-call-stack-exceeded-wip", + "quakejs-repak": "0.0.1" + } +}