diff --git a/README.md b/README.md index 169004288aa..ac82b6642bc 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ yum install -y \ zlib zlib-devel ``` +> nginx 最终安装在 `/home/jsproxy/openresty` 下，不会和系统已有的冲突。 + + ## 测试 启动服务： diff --git a/api.conf b/api.conf index 056f19b50a3..ceec38a31a9 100644 --- a/api.conf +++ b/api.conf @@ -15,7 +15,7 @@ location = /preflight { internal; more_set_headers 'access-control-allow-origin: *' - 'access-control-allow-methods: GET,POST,PUT,PATCH,DELETE,HEAD,OPTIONS' + 'access-control-allow-methods: GET,POST,PUT,PATCH,TRACE,DELETE,HEAD,OPTIONS' 'access-control-allow-headers: --url,--referer,--cookie,--origin,--ext,--aceh,--ver,accept,accept-charset,accept-encoding,accept-language,accept-datetime,authorization,cache-control,content-length,content-type,date,if-match,if-modified-since,if-none-match,if-range,if-unmodified-since,max-forwards,pragma,range,te,upgrade,upgrade-insecure-requests,x-requested-with,chrome-proxy' 'access-control-max-age: 1728000' ; @@ -29,40 +29,7 @@ location = /http { rewrite ^ /preflight; } - # decode req headers - access_by_lua_block { - local hdrs, err = ngx.req.get_headers() - local extHdrs - - for k, v in pairs(hdrs) do - if k:sub(1, 2) ~= '--' then - goto continue - end - - ngx.req.clear_header(k) - k = k:sub(3) - - if k == 'url' then - ngx.var._url = v - elseif k == 'ver' then - ngx.var._ver = v - elseif k == 'aceh' then - ngx.ctx._aceh = 1 - elseif k == 'ext' then - extHdrs = require('cjson').decode(v) - else - ngx.req.set_header(k, v) - end - - ::continue:: - end - - if extHdrs then - for k, v in pairs(extHdrs) do - ngx.req.set_header(k, v) - end - end - } + access_by_lua_file ../lua/http-dec-req-hdr.lua; proxy_cache my_cache; proxy_pass $_url; @@ -73,89 +40,23 @@ location = /http { 'content-security-policy-report-only' 'x-frame-options' ; - - # encode res headers - header_filter_by_lua_block { - local expose = '*' - local detail = (ngx.ctx._aceh == 1) - local vary = '--url' - - local h, err = ngx.resp.get_headers() - for k, v in pairs(h) do - if - -- headers to escape -- - k == 'access-control-allow-origin' or - k == 'access-control-expose-headers' or - k == 'location' or - k == 'set-cookie' - then - if type(v) == 'table' then - for i = 1, #v do - local x = i .. '-' .. k - ngx.header[x] = v[i] - - if detail then - expose = expose .. ',' .. x - end - end - else - local x = '--' .. k - ngx.header[x] = v - - if detail then - expose = expose .. ',' .. x - end - end - ngx.header[k] = nil - - elseif k == 'vary' then - if type(v) == 'table' then - vary = vary .. ', ' .. table.concat(v, ', ') - else - vary = vary .. ', ' .. v - end - - elseif detail and - -- not simple header -- - k ~= 'cache-control' and - k ~= 'cache-language' and - k ~= 'content-type' and - k ~= 'expires' and - k ~= 'last-modified' and - k ~= 'pragma' - then - expose = expose .. ',' .. k - end - end - - if detail then - expose = expose .. ',--s' - ngx.header['--t'] = '1' - end - - ngx.header['access-control-expose-headers'] = expose - ngx.header['access-control-allow-origin'] = '*' - ngx.header['vary'] = vary - ngx.header['--s'] = ngx.status - ngx.status = 200 - } + header_filter_by_lua_file ../lua/http-enc-res-hdr.lua; } # WebSocket Proxy location = /ws { - access_by_lua_block { - local query, err = ngx.req.get_uri_args() - - for k, v in pairs(query) do - if k == 'url__' then - ngx.var._url = v - elseif k == 'ver__' then - ngx.var._ver = v - else - ngx.req.set_header(k, v) - end - end - } + access_by_lua_file ../lua/ws-dec-req-hdr.lua; proxy_pass $_url; } + + +location = /traff { + content_by_lua_block { + ngx.say(ngx.shared.traff:get('stat')) + } + more_set_headers + 'access-control-allow-origin: *' + 'cache-control: no-cache' + ; +} \ No newline at end of file diff --git a/log-svc/README.md b/log-svc/README.md new file mode 100644 index 00000000000..2ebefd554e4 --- /dev/null +++ b/log-svc/README.md @@ -0,0 +1,21 @@ +nginx 日志备份服务 + +## 说明 + +nginx 长时间运行会导致日志文件过大，该服务定期备份日志到 `backup` 目录，并进行压缩。 + + +## 依赖 + +用到了 `brotli` 压缩工具，执行 `setup-brotli.sh` 安装。 + +最终安装在 `/home/jsproxy/tools/brotli`。 + + +## 启动 + +```bash +./svc.sh & +``` + +使用 `jsproxy` 用户运行，无需 `root`。 \ No newline at end of file diff --git a/log-svc/backup.sh b/log-svc/backup.sh new file mode 100755 index 00000000000..42acc734383 --- /dev/null +++ b/log-svc/backup.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# 功能：备份 nginx 日志到 backup 目录 + +SVC_DIR=/home/jsproxy/server +LOG_DIR=$SVC_DIR/nginx/logs + +LOG_FILE=$LOG_DIR/proxy.log +LOG_SIZE=$(( 32 * 1024 * 1024 )) + +ERR_FILE=$LOG_DIR/error.log +ERR_SIZE=$(( 1 * 1024 * 1024 * 1024 )) + + +# error.log 达到 ERR_SIZE，开始备份（目前只清理） +errsize=$(stat --printf=%s $ERR_FILE) +if (( $errsize >= $ERR_SIZE )); then + echo > $ERR_FILE +fi + +# proxy.log 达到 LOG_SIZE，开始备份 +logsize=$(stat --printf=%s $LOG_FILE) +if (( $logsize < $LOG_SIZE )); then + exit +fi + +logtime=$(date "+%Y-%m-%d-%H-%M-%S") +logfile=$SVC_DIR/log-svc/backup/$logtime.log + +# +# 先移走日志文件，然后创建新的日志文件，通知 nginx 重新打开 +# https://www.nginx.com/resources/wiki/start/topics/examples/logrotation/ +# +mv $LOG_FILE $logfile +touch $LOG_FILE +kill -USR1 $(< $LOG_DIR/nginx.pid) +sleep 1 + +# +# 日志压缩 +# 根据实际情况调整策略，在不影响系统的前提下，充分利用剩余 CPU +# 可尝试其他工具（例如 7z），在开销和效果之间找一个平衡点 +# +echo "compress $logtime ($logsize bytes)" + +if (( $logsize > 100 * 1024 * 1024 )); then + # 日志较大，使用快速压缩 + nice -n 19 \ + gzip $logfile +else + # 日志不大，使用高强度压缩 + nice -n 19 \ + ~/tools/brotli $logfile --rm +fi + +echo "done" \ No newline at end of file diff --git a/log-svc/backup/README.md b/log-svc/backup/README.md new file mode 100644 index 00000000000..51e70b4b35f --- /dev/null +++ b/log-svc/backup/README.md @@ -0,0 +1 @@ +该目录存放临时备份的日志。 \ No newline at end of file diff --git a/log-svc/setup-brotli.sh b/log-svc/setup-brotli.sh new file mode 100644 index 00000000000..cca8aa2ca74 --- /dev/null +++ b/log-svc/setup-brotli.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# 功能：安装 brotli 压缩工具 +# 依赖：cmake（yum install -y cmake） + +git clone --depth 1 https://github.com/google/brotli.git +cd brotli + +./configure-cmake +make + +mkdir -p ~/tools +mv brotli ~/tools + +cd .. +rm -rf brotli \ No newline at end of file diff --git a/log-svc/svc.sh b/log-svc/svc.sh new file mode 100755 index 00000000000..eecd5eed579 --- /dev/null +++ b/log-svc/svc.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# 功能：定时调用 backup.sh + +echo "log svc running" + +# 也可用 crontab +while true +do + ./backup.sh + sleep 60 +done \ No newline at end of file diff --git a/lua/g.lua b/lua/g.lua new file mode 100644 index 00000000000..1334d6ab394 --- /dev/null +++ b/lua/g.lua @@ -0,0 +1,37 @@ +local _M = {} +local traff = ngx.shared.traff + +local nReq = 0 + + +function _M.inc() + nReq = nReq + 1 +end + +function _M.syn() + traff:incr('nReq', nReq) + nReq = 0 +end + +function _M.update() + local nReq = traff:get('nReq') + traff:set('nReq', 0) + return nReq +end + +function _M.getStat() + return traff:get('stat') +end + +function _M.setStat(stat) + return traff:set('stat', stat) +end + +function _M.reset() + if traff:get('nReq') == nil then + traff:add('nReq', 0) + traff:add('stat', '') + end +end + +return _M \ No newline at end of file diff --git a/lua/http-dec-req-hdr.lua b/lua/http-dec-req-hdr.lua new file mode 100644 index 00000000000..0ddea37cc82 --- /dev/null +++ b/lua/http-dec-req-hdr.lua @@ -0,0 +1,34 @@ +-- 功能：还原 HTTP 请求头 +-- 阶段：access_by_lua + +local hdrs, err = ngx.req.get_headers() +local extHdrs + +for k, v in pairs(hdrs) do + if k:sub(1, 2) ~= '--' then + goto continue + end + + ngx.req.clear_header(k) + k = k:sub(3) + + if k == 'url' then + ngx.var._url = v + elseif k == 'ver' then + ngx.var._ver = v + elseif k == 'aceh' then + ngx.ctx._aceh = 1 + elseif k == 'ext' then + extHdrs = require('cjson').decode(v) + else + ngx.req.set_header(k, v) + end + + ::continue:: +end + +if extHdrs then + for k, v in pairs(extHdrs) do + ngx.req.set_header(k, v) + end +end \ No newline at end of file diff --git a/lua/http-enc-res-hdr.lua b/lua/http-enc-res-hdr.lua new file mode 100644 index 00000000000..e67cec959a8 --- /dev/null +++ b/lua/http-enc-res-hdr.lua @@ -0,0 +1,74 @@ +-- 功能：编码 HTTP 返回头 +-- 阶段：header_filter_by_lua +-- 备注： +-- aceh = HTTP 返回头的 access-control-expose-headers 字段 + + +-- 无论浏览器是否支持，aceh 始终包含 * +local expose = '*' + +-- 该值为 true 表示浏览器不支持 aceh: *，需返回详细的头部列表 +local detail = (ngx.ctx._aceh == 1) + +-- 由于接口路径固定，为避免被缓存，以请求头的 --url 值区分缓存 +local vary = '--url' + +local h, err = ngx.resp.get_headers() +for k, v in pairs(h) do + if + -- 这些头有特殊意义，需要转义 -- + k == 'access-control-allow-origin' or + k == 'access-control-expose-headers' or + k == 'location' or + k == 'set-cookie' + then + if type(v) == 'table' then + for i = 1, #v do + local x = i .. '-' .. k + ngx.header[x] = v[i] + + if detail then + expose = expose .. ',' .. x + end + end + else + local x = '--' .. k + ngx.header[x] = v + + if detail then + expose = expose .. ',' .. x + end + end + ngx.header[k] = nil + + elseif k == 'vary' then + if type(v) == 'table' then + vary = vary .. ',' .. table.concat(v, ',') + else + vary = vary .. ',' .. v + end + + elseif detail and + -- 非简单头无法被 fetch 读取，需添加到 aceh 列表 -- + -- https://developer.mozilla.org/en-US/docs/Glossary/Simple_response_header + k ~= 'cache-control' and + k ~= 'content-language' and + k ~= 'content-type' and + k ~= 'expires' and + k ~= 'last-modified' and + k ~= 'pragma' + then + expose = expose .. ',' .. k + end +end + +if detail then + expose = expose .. ',--s' + ngx.header['--t'] = '1' +end + +ngx.header['access-control-expose-headers'] = expose +ngx.header['access-control-allow-origin'] = '*' +ngx.header['vary'] = vary +ngx.header['--s'] = ngx.status +ngx.status = 200 \ No newline at end of file diff --git a/lua/init.lua b/lua/init.lua new file mode 100644 index 00000000000..738279e36b9 --- /dev/null +++ b/lua/init.lua @@ -0,0 +1,6 @@ +local traff = ngx.shared.traff + +if traff:get('nReq') == nil then + traff:add('nReq', 0) + traff:add('stat', '') +end \ No newline at end of file diff --git a/lua/worker.lua b/lua/worker.lua new file mode 100644 index 00000000000..604615a96a1 --- /dev/null +++ b/lua/worker.lua @@ -0,0 +1,70 @@ +local g = require('g') + +-- run in master worker +if ngx.worker.id() ~= 0 then + return +end + +local function getDevTraffic(dev) + -- 0 1 2 3 4 5 6 7 + -- eth0: bytes packets errs drop fifo frame compressed multicast + -- bytes packets errs drop fifo colls carrier compressed + local regex = dev .. ':%s+(%d+)%s+' .. ('%d+%s+'):rep(7) .. '(%d+)' + + local lastRxBytes = 0 + local lastTxBytes = 0 + + return function(str) + local + sRxBytes, + sTxBytes + = str:match(regex) + + if sTxBytes == nil then + return '0,0' + end + + local nRxBytes = tonumber(sRxBytes) + local nTxBytes = tonumber(sTxBytes) + + local rxBPS = nRxBytes - lastRxBytes + local txBPS = nTxBytes - lastTxBytes + + lastRxBytes = nRxBytes + lastTxBytes = nTxBytes + + return rxBPS .. ',' .. txBPS + end +end + + +local fileStat = io.open('/proc/net/dev') +if fileStat == nil then + ngx.log(ngx.ERR, 'open `/proc/net/dev` fail') + return +end + +local firstRun = true +local getDevTraffic = getDevTraffic('eth0') + + +local function updateTraffic() + local r, err = fileStat:seek('set') + local out = fileStat:read('*all') + + local traffDev = getDevTraffic(out) + + if firstRun then + firstRun = false + return + end + + g.syn() + + local traffHttp = g.update() + local stat = traffDev .. ',' .. traffHttp + + g.setStat(stat) +end + +ngx.timer.every(1, updateTraffic) \ No newline at end of file diff --git a/lua/ws-dec-req-hdr.lua b/lua/ws-dec-req-hdr.lua new file mode 100644 index 00000000000..c1597513660 --- /dev/null +++ b/lua/ws-dec-req-hdr.lua @@ -0,0 +1,15 @@ +-- 功能：还原 WebSocket 的 HTTP 请求头 +-- 阶段：access_by_lua +-- 备注：JS 无法设置 ws 的头部，因此信息存储于 query + +local query, err = ngx.req.get_uri_args() + +for k, v in pairs(query) do + if k == 'url__' then + ngx.var._url = v + elseif k == 'ver__' then + ngx.var._ver = v + else + ngx.req.set_header(k, v) + end +end \ No newline at end of file diff --git a/nginx.conf b/nginx.conf index e5a607ecb69..ffca5730806 100644 --- a/nginx.conf +++ b/nginx.conf @@ -36,9 +36,9 @@ http { proxy_cache_path cache levels=1:2 - keys_zone=my_cache:8m - max_size=10g - inactive=6h + keys_zone=my_cache:32m + max_size=20g + inactive=24h use_temp_path=off ; proxy_http_version 1.1; @@ -49,6 +49,14 @@ http { proxy_busy_buffers_size 64k; proxy_send_timeout 10s; + # traffic statistics + lua_shared_dict traff 16k; + + lua_package_path ./lua/?.lua; + init_by_lua_file ../lua/init.lua; + init_worker_by_lua_file ../lua/worker.lua; + log_by_lua require('g').inc(); + map $http_origin $_origin_id { include allowed-sites.conf; } diff --git a/setup-ipset.sh b/setup-ipset.sh index 4e7667e9ac1..f34c3d59577 100755 --- a/setup-ipset.sh +++ b/setup-ipset.sh @@ -1,6 +1,7 @@ -# run as root +# 需要 root 运行 ipset create ngx-ban-dstip hash:net +# 该策略对 jsproxy 用户的所有程序都生效 iptables \ -A OUTPUT \ -p tcp --syn \ @@ -31,3 +32,6 @@ REV_NET=( for v in ${REV_NET[@]}; do ipset add ngx-ban-dstip $v done + +# 可屏蔽更多的网段： +# ipset add ngx-ban-dstip xxx \ No newline at end of file diff --git a/setup-nginx.sh b/setup-nginx.sh index 1a400d05292..bad23588b9b 100755 --- a/setup-nginx.sh +++ b/setup-nginx.sh @@ -1,4 +1,4 @@ -# +# 无需 root 运行 curl -O https://openresty.org/download/openresty-1.15.8.1rc1.tar.gz tar zxvf openresty-* cd openresty-* @@ -6,7 +6,6 @@ cd openresty-* ./configure \ --with-http_v2_module \ --with-http_ssl_module \ - --with-pcre-jit \ --prefix=/home/jsproxy/openresty make diff --git a/upload.sh b/upload.sh index b5d4ddbb2cc..3d58a973a87 100755 --- a/upload.sh +++ b/upload.sh @@ -1,3 +1,6 @@ +#!/usr/bin/env bash +# 功能：同步文件到所有节点，并重启服务 + HOST=etherdream.com NODE=( node-aliyun-hk @@ -8,7 +11,7 @@ for v in ${NODE[@]}; do echo "$v upload ..." rsync . jsproxy@$v.$HOST:server \ - -r \ + -r -p \ --exclude='nginx/cache/*' \ --exclude='nginx/logs/*'