From 9e9be9a3a7c2b9fbd774262da96f8e62132d3fc3 Mon Sep 17 00:00:00 2001 From: Sansui233 Date: Thu, 26 Nov 2020 15:46:51 +0800 Subject: [PATCH] optimize memory --- README_NEW.md | 6 ++- internal/app/task.go | 74 ++++++++++++++++++++++++++++------- main.go | 7 ++-- pkg/healthcheck/delaycheck.go | 7 ++-- pkg/proxy/base.go | 9 ++--- pkg/proxy/convert.go | 2 +- pkg/proxy/shadowsocks.go | 3 +- pkg/proxy/shadowsocksr.go | 37 +++++++----------- pkg/proxy/trojan.go | 5 +-- pkg/proxy/vmess.go | 16 ++++---- 10 files changed, 102 insertions(+), 64 deletions(-) diff --git a/README_NEW.md b/README_NEW.md index a63f7a0b..6eb1625e 100644 --- a/README_NEW.md +++ b/README_NEW.md @@ -4,14 +4,16 @@ Clash客户端支持： - 不支持ClashXR与ClashR等非原生Clash Core客户端。 TODO +- [ ] 考虑更换grpool。 + 节点过多时会卡在节点检测上，并且本地无法复现。本以为是内存问题，但观察内存时发现mem和cpu的占用实际上都非常少。TCP连接关闭正常并非网络阻塞。 - [ ] 储存日志。 -- [ ] external api for config update ## New 2020-11-26 - 修复了一个导致vmess h2解析错误的bug= = -- 增强对cf的js重写的破解。目前看来还需要大量的实例完善此功能 +- 显著增强对cf的js重写的破解。目前看来还需要大量的实例完善此功能 +- 优化命名流程，结构体对齐调整，优化内存占用（时间与算力换空间）。 2020-11-25 - 增加tg channel文件的抓取 diff --git a/internal/app/task.go b/internal/app/task.go index d0dcc84c..5f18047d 100644 --- a/internal/app/task.go +++ b/internal/app/task.go @@ -32,31 +32,29 @@ func CrawlGo() { cache.LastCrawlTime = "抓取中，已载入上次数据库数据" fmt.Println("Database: loaded") } - proxies = append(proxies, db_proxies...) + proxies = uniqAppend(proxies, db_proxies) go func() { wg.Wait() close(pc) }() // Note: 为何并发？可以一边抓取一边读取而非抓完再读 - for node := range pc { // Note: pc关闭后不能发送数据可以读取剩余数据 - if node != nil { - proxies = append(proxies, node) + for p := range pc { // Note: pc关闭后不能发送数据可以读取剩余数据 + if p != nil { + proxies = uniqAppendProxy(proxies, p) } } - // 节点衍生并去重 - proxies = proxies.Deduplication().Derive() - log.Println("CrawlGo unique node count:", len(proxies)) + // 节点衍生并去重 -> 去重在上面做了，衍生的必要性不太明白 + //proxies = proxies.Deduplication().Derive() + log.Println("CrawlGo unique proxy count:", len(proxies)) + // 去除Clash(windows)不支持的节点 proxies = provider.Clash{ provider.Base{ Proxies: &proxies, }, }.CleanProxies() - log.Println("CrawlGo clash supported node count:", len(proxies)) - // 重命名节点名称为类似US_01的格式，并按国家排序 - proxies.NameSetCounrty().Sort().NameAddIndex() //由于原作停更，暂不加.NameAddTG()，如被认为有版权问题请告知 - log.Println("Proxy rename DONE!") + log.Println("CrawlGo clash supported proxy count:", len(proxies)) cache.SetProxies("allproxies", proxies) cache.AllProxiesCount = proxies.Len() @@ -71,11 +69,25 @@ func CrawlGo() { log.Println("TrojanProxiesCount:", cache.TrojanProxiesCount) cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05") - // 节点可用性检测 + // 节点可用性检测，使用batchsize不能降低内存占用，只是为了看性能 log.Println("Now proceed proxy health check...") + b := 1000 + round := len(proxies) / b + okproxies := make(proxy.ProxyList, 0) + for i := 0; i < round; i++ { + okproxies = append(okproxies, healthcheck.CleanBadProxiesWithGrpool(proxies[i*b:(i+1)*b])...) + log.Println("Checking round:", i) + } + okproxies = append(okproxies, healthcheck.CleanBadProxiesWithGrpool(proxies[round*b:])...) + proxies = okproxies + proxies = healthcheck.CleanBadProxiesWithGrpool(proxies) - log.Println("CrawlGo clash usable node count:", len(proxies)) - proxies.NameReIndex() //由于原作停更，暂不加.NameAddTG()，如被认为有版权问题请告知 + log.Println("CrawlGo clash usable proxy count:", len(proxies)) + + // 重命名节点名称为类似US_01的格式，并按国家排序 + proxies.NameSetCounrty().Sort().NameAddIndex() + //proxies.NameReIndex() + log.Println("Proxy rename DONE!") // 可用节点存储 cache.SetProxies("proxies", proxies) @@ -126,3 +138,37 @@ func SpeedTest(proxies proxy.ProxyList) { cache.IsSpeedTest = "未开启" } } + +// Append unique new proxies to old proxy.ProxyList +func uniqAppend(pl proxy.ProxyList, new proxy.ProxyList) proxy.ProxyList { + if len(new) == 0 { + return pl + } + if len(pl) == 0 { + return new + } + for _, p := range new { + for i, _ := range pl { + if pl[i].Identifier() == p.Identifier() { + continue + } + } + pl = append(pl, p) + } + return pl +} + +// Append unique new proxies to old proxy.ProxyList +func uniqAppendProxy(pl proxy.ProxyList, new proxy.Proxy) proxy.ProxyList { + if len(pl) == 0 { + pl = append(pl, new) + return pl + } + for i, _ := range pl { + if pl[i].Identifier() == new.Identifier() { + return pl + } + } + pl = append(pl, new) + return pl +} diff --git a/main.go b/main.go index 0c2978e5..5eadf484 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "flag" "fmt" - "net/http" _ "net/http/pprof" "os" @@ -17,9 +16,9 @@ import ( var configFilePath = "" func main() { - go func() { - http.ListenAndServe("0.0.0.0:6060", nil) - }() + //go func() { + // http.ListenAndServe("0.0.0.0:6060", nil) + //}() flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml") flag.Parse() diff --git a/pkg/healthcheck/delaycheck.go b/pkg/healthcheck/delaycheck.go index 3745837e..ac655b60 100644 --- a/pkg/healthcheck/delaycheck.go +++ b/pkg/healthcheck/delaycheck.go @@ -63,9 +63,10 @@ func CleanBadProxiesWithGrpool(proxies []proxy.Proxy) (cproxies []proxy.Proxy) { case <-done: cproxies = make(proxy.ProxyList, 0, 500) // 定义返回的proxylist // check usable proxy - for _, p := range proxies { - if _, ok := okMap[p.Identifier()]; ok { - cproxies = append(cproxies, p.Clone()) + for i, _ := range proxies { + if _, ok := okMap[proxies[i].Identifier()]; ok { + //cproxies = append(cproxies, p) // 没必要Clone因为range时就复制了一份 + cproxies = append(cproxies, proxies[i]) // 使用对GC非常不友好的指针看看会不会有事 } } return diff --git a/pkg/proxy/base.go b/pkg/proxy/base.go index 2fcb2572..537ab01c 100644 --- a/pkg/proxy/base.go +++ b/pkg/proxy/base.go @@ -9,16 +9,13 @@ import ( type Base struct { Name string `yaml:"name" json:"name" gorm:"index"` Server string `yaml:"server" json:"server" gorm:"index"` - Port int `yaml:"port" json:"port" gorm:"index"` Type string `yaml:"type" json:"type" gorm:"index"` - UDP bool `yaml:"udp,omitempty" json:"udp,omitempty"` Country string `yaml:"country,omitempty" json:"country,omitempty" gorm:"index"` - // 这个单词的原作者拼写是错误的，但我不想改了，我也没有早点发现这件事，在写where查询老写错，非常的无奈 - Useable bool `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"` + Port int `yaml:"port" json:"port" gorm:"index"` + UDP bool `yaml:"udp,omitempty" json:"udp,omitempty"` + Useable bool `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"` } -// Note: Go只有值传递，必需传入指针才能改变传入的结构体 - // TypeName() Get specific proxy type func (b *Base) TypeName() string { if b.Type == "" { diff --git a/pkg/proxy/convert.go b/pkg/proxy/convert.go index 9a7d06d2..1534b750 100644 --- a/pkg/proxy/convert.go +++ b/pkg/proxy/convert.go @@ -26,7 +26,7 @@ func Convert2SSR(p Proxy) (ssr *ShadowsocksR, err error) { Cipher: ss.Cipher, Protocol: "origin", Obfs: "plain", - Group: "proxy.tgbot.co", + Group: "", }, nil } return nil, ErrorTypeCanNotConvert diff --git a/pkg/proxy/shadowsocks.go b/pkg/proxy/shadowsocks.go index c1cac2f2..19c4e19f 100644 --- a/pkg/proxy/shadowsocks.go +++ b/pkg/proxy/shadowsocks.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" "net" "net/url" "regexp" @@ -143,7 +142,7 @@ func ParseSSLink(link string) (*Shadowsocks, error) { return &Shadowsocks{ Base: Base{ - Name: strconv.Itoa(rand.Int()), + Name: "", Server: server, Port: port, Type: "ss", diff --git a/pkg/proxy/shadowsocksr.go b/pkg/proxy/shadowsocksr.go index f2dadc0c..b6bacf8d 100644 --- a/pkg/proxy/shadowsocksr.go +++ b/pkg/proxy/shadowsocksr.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" "net" "net/url" "regexp" @@ -70,7 +69,7 @@ func (ssr ShadowsocksR) Link() (link string) { query := url.Values{} query.Add("obfsparam", tool.Base64EncodeString(ssr.ObfsParam, true)) query.Add("protoparam", tool.Base64EncodeString(ssr.ProtocolParam, true)) - query.Add("remarks", tool.Base64EncodeString(ssr.Name, true)) + //query.Add("remarks", tool.Base64EncodeString(ssr.Name, true)) query.Add("group", tool.Base64EncodeString("proxypoolss.herokuapp.com", true)) payload = tool.Base64EncodeString(fmt.Sprintf("%s/?%s", payload, query.Encode()), true) return fmt.Sprintf("ssr://%s", payload) @@ -113,18 +112,18 @@ func ParseSSRLink(link string) (*ShadowsocksR, error) { moreInfo, _ := url.ParseQuery(infoPayload[1]) // remarks - remarks := moreInfo.Get("remarks") - remarks, err = tool.Base64DecodeString(remarks) - if err != nil { - remarks = "" - err = nil - } - if strings.ContainsAny(remarks, "\t\r\n ") { - remarks = strings.ReplaceAll(remarks, "\t", "") - remarks = strings.ReplaceAll(remarks, "\r", "") - remarks = strings.ReplaceAll(remarks, "\n", "") - remarks = strings.ReplaceAll(remarks, " ", "") - } + //remarks := moreInfo.Get("remarks") + //remarks, err = tool.Base64DecodeString(remarks) + //if err != nil { + // remarks = "" + // err = nil + //} + //if strings.ContainsAny(remarks, "\t\r\n ") { + // remarks = strings.ReplaceAll(remarks, "\t", "") + // remarks = strings.ReplaceAll(remarks, "\r", "") + // remarks = strings.ReplaceAll(remarks, "\n", "") + // remarks = strings.ReplaceAll(remarks, " ", "") + //} // protocol param protocolParam, err := tool.Base64DecodeString(moreInfo.Get("protoparam")) @@ -150,15 +149,9 @@ func ParseSSRLink(link string) (*ShadowsocksR, error) { obfs = strings.ReplaceAll(obfs, "_compatible", "") } - //group, err := tool.Base64DecodeString(moreInfo.Get("group")) - //if err != nil { - // group = "" - //} - group := "" - return &ShadowsocksR{ Base: Base{ - Name: remarks + "_" + strconv.Itoa(rand.Int()), + Name: "", Server: server, Port: port, Type: "ssr", @@ -169,7 +162,7 @@ func ParseSSRLink(link string) (*ShadowsocksR, error) { ProtocolParam: protocolParam, Obfs: obfs, ObfsParam: obfsParam, - Group: group, + Group: "", }, nil } diff --git a/pkg/proxy/trojan.go b/pkg/proxy/trojan.go index fab75e68..bcd75f67 100644 --- a/pkg/proxy/trojan.go +++ b/pkg/proxy/trojan.go @@ -3,7 +3,6 @@ package proxy import ( "encoding/json" "errors" - "math/rand" "net" "net/url" "regexp" @@ -138,15 +137,15 @@ func ParseTrojanLink(link string) (*Trojan, error) { return &Trojan{ Base: Base{ - Name: strconv.Itoa(rand.Int()), + Name: "", Server: server, Port: port, Type: "trojan", }, Password: password, ALPN: alpn, - UDP: true, SNI: host, + UDP: true, SkipCertVerify: true, }, nil } diff --git a/pkg/proxy/vmess.go b/pkg/proxy/vmess.go index 617b7b34..c35c7d2b 100644 --- a/pkg/proxy/vmess.go +++ b/pkg/proxy/vmess.go @@ -25,14 +25,14 @@ type Vmess struct { UUID string `yaml:"uuid" json:"uuid"` AlterID int `yaml:"alterId" json:"alterId"` Cipher string `yaml:"cipher" json:"cipher"` - TLS bool `yaml:"tls,omitempty" json:"tls,omitempty"` Network string `yaml:"network,omitempty" json:"network,omitempty"` - HTTPOpts HTTPOptions `yaml:"http-opts,omitempty" json:"http-opts,omitempty"` - HTTP2Opts HTTP2Options `yaml:"h2-opts,omitempty" json:"h2-opts,omitempty"` WSPath string `yaml:"ws-path,omitempty" json:"ws-path,omitempty"` + ServerName string `yaml:"servername,omitempty" json:"servername,omitempty"` WSHeaders map[string]string `yaml:"ws-headers,omitempty" json:"ws-headers,omitempty"` + HTTPOpts HTTPOptions `yaml:"http-opts,omitempty" json:"http-opts,omitempty"` + HTTP2Opts HTTP2Options `yaml:"h2-opts,omitempty" json:"h2-opts,omitempty"` + TLS bool `yaml:"tls,omitempty" json:"tls,omitempty"` SkipCertVerify bool `yaml:"skip-cert-verify,omitempty" json:"skip-cert-verify,omitempty"` - ServerName string `yaml:"servername,omitempty" json:"servername,omitempty"` } type HTTPOptions struct { @@ -314,7 +314,7 @@ func ParseVmessLink(link string) (*Vmess, error) { return &Vmess{ Base: Base{ - Name: vmessJson.Ps + "_" + strconv.Itoa(rand.Int()), + Name: "", Server: vmessJson.Add, Port: vmessJson.Port, Type: "vmess", @@ -323,14 +323,14 @@ func ParseVmessLink(link string) (*Vmess, error) { UUID: vmessJson.Id, AlterID: alterId, Cipher: "auto", - TLS: tls, Network: vmessJson.Net, HTTPOpts: httpOpt, HTTP2Opts: h2Opt, WSPath: vmessJson.Path, WSHeaders: wsHeaders, - SkipCertVerify: true, ServerName: vmessJson.Host, + TLS: tls, + SkipCertVerify: true, }, nil } } @@ -385,6 +385,8 @@ func mapStrInter2VmessLinkJson(jsn map[string]interface{}) (vmessLinkJson, error default: vmessVal.Field(i).SetInt(443) } + } else if strings.ToLower(tag[0]) == "ps" { + continue } else { // set string in other fields switch jsnVal.(type) { case string: