optimize memory

README_NEW.md

@@ -4,14 +4,16 @@ Clash客户端支持：
 - 不支持ClashXR与ClashR等非原生Clash Core客户端。
 
 TODO
+- [ ] 考虑更换grpool。
+  节点过多时会卡在节点检测上，并且本地无法复现。本以为是内存问题，但观察内存时发现mem和cpu的占用实际上都非常少。TCP连接关闭正常并非网络阻塞。
 - [ ] 储存日志。
-- [ ] external api for config update
 
 ## New
 
 2020-11-26
 - 修复了一个导致vmess h2解析错误的bug= =
-- 增强对cf的js重写的破解。目前看来还需要大量的实例完善此功能
+- 显著增强对cf的js重写的破解。目前看来还需要大量的实例完善此功能
+- 优化命名流程，结构体对齐调整，优化内存占用（时间与算力换空间）。
 
 2020-11-25
 - 增加tg channel文件的抓取

internal/app/task.go

@@ -32,31 +32,29 @@ func CrawlGo() {
 		cache.LastCrawlTime = "抓取中，已载入上次数据库数据"
 		fmt.Println("Database: loaded")
 	}
-	proxies = append(proxies, db_proxies...)
+	proxies = uniqAppend(proxies, db_proxies)
 
 	go func() {
 		wg.Wait()
 		close(pc)
 	}() // Note: 为何并发？可以一边抓取一边读取而非抓完再读
-	for node := range pc { // Note: pc关闭后不能发送数据可以读取剩余数据
-		if node != nil {
-			proxies = append(proxies, node)
+	for p := range pc { // Note: pc关闭后不能发送数据可以读取剩余数据
+		if p != nil {
+			proxies = uniqAppendProxy(proxies, p)
 		}
 	}
 
-	// 节点衍生并去重
-	proxies = proxies.Deduplication().Derive()
-	log.Println("CrawlGo unique node count:", len(proxies))
+	// 节点衍生并去重 -> 去重在上面做了，衍生的必要性不太明白
+	//proxies = proxies.Deduplication().Derive()
+	log.Println("CrawlGo unique proxy count:", len(proxies))
+
 	// 去除Clash(windows)不支持的节点
 	proxies = provider.Clash{
 		provider.Base{
 			Proxies: &proxies,
 		},
 	}.CleanProxies()
-	log.Println("CrawlGo clash supported node count:", len(proxies))
-	// 重命名节点名称为类似US_01的格式，并按国家排序
-	proxies.NameSetCounrty().Sort().NameAddIndex() //由于原作停更，暂不加.NameAddTG()，如被认为有版权问题请告知
-	log.Println("Proxy rename DONE!")
+	log.Println("CrawlGo clash supported proxy count:", len(proxies))
 
 	cache.SetProxies("allproxies", proxies)
 	cache.AllProxiesCount = proxies.Len()
@@ -71,11 +69,25 @@ func CrawlGo() {
 	log.Println("TrojanProxiesCount:", cache.TrojanProxiesCount)
 	cache.LastCrawlTime = time.Now().In(location).Format("2006-01-02 15:04:05")
 
-	// 节点可用性检测
+	// 节点可用性检测，使用batchsize不能降低内存占用，只是为了看性能
 	log.Println("Now proceed proxy health check...")
+	b := 1000
+	round := len(proxies) / b
+	okproxies := make(proxy.ProxyList, 0)
+	for i := 0; i < round; i++ {
+		okproxies = append(okproxies, healthcheck.CleanBadProxiesWithGrpool(proxies[i*b:(i+1)*b])...)
+		log.Println("Checking round:", i)
+	}
+	okproxies = append(okproxies, healthcheck.CleanBadProxiesWithGrpool(proxies[round*b:])...)
+	proxies = okproxies
+
 	proxies = healthcheck.CleanBadProxiesWithGrpool(proxies)
-	log.Println("CrawlGo clash usable node count:", len(proxies))
-	proxies.NameReIndex() //由于原作停更，暂不加.NameAddTG()，如被认为有版权问题请告知
+	log.Println("CrawlGo clash usable proxy count:", len(proxies))
+
+	// 重命名节点名称为类似US_01的格式，并按国家排序
+	proxies.NameSetCounrty().Sort().NameAddIndex()
+	//proxies.NameReIndex()
+	log.Println("Proxy rename DONE!")
 
 	// 可用节点存储
 	cache.SetProxies("proxies", proxies)
@@ -126,3 +138,37 @@ func SpeedTest(proxies proxy.ProxyList) {
 		cache.IsSpeedTest = "未开启"
 	}
 }
+
+// Append unique new proxies to old proxy.ProxyList
+func uniqAppend(pl proxy.ProxyList, new proxy.ProxyList) proxy.ProxyList {
+	if len(new) == 0 {
+		return pl
+	}
+	if len(pl) == 0 {
+		return new
+	}
+	for _, p := range new {
+		for i, _ := range pl {
+			if pl[i].Identifier() == p.Identifier() {
+				continue
+			}
+		}
+		pl = append(pl, p)
+	}
+	return pl
+}
+
+// Append unique new proxies to old proxy.ProxyList
+func uniqAppendProxy(pl proxy.ProxyList, new proxy.Proxy) proxy.ProxyList {
+	if len(pl) == 0 {
+		pl = append(pl, new)
+		return pl
+	}
+	for i, _ := range pl {
+		if pl[i].Identifier() == new.Identifier() {
+			return pl
+		}
+	}
+	pl = append(pl, new)
+	return pl
+}

main.go

@@ -3,7 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"net/http"
 	_ "net/http/pprof"
 	"os"
 
@@ -17,9 +16,9 @@ import (
 var configFilePath = ""
 
 func main() {
-	go func() {
-		http.ListenAndServe("0.0.0.0:6060", nil)
-	}()
+	//go func() {
+	//	http.ListenAndServe("0.0.0.0:6060", nil)
+	//}()
 
 	flag.StringVar(&configFilePath, "c", "", "path to config file: config.yaml")
 	flag.Parse()

pkg/healthcheck/delaycheck.go

@@ -63,9 +63,10 @@ func CleanBadProxiesWithGrpool(proxies []proxy.Proxy) (cproxies []proxy.Proxy) {
 		case <-done:
 			cproxies = make(proxy.ProxyList, 0, 500) // 定义返回的proxylist
 			// check usable proxy
-			for _, p := range proxies {
-				if _, ok := okMap[p.Identifier()]; ok {
-					cproxies = append(cproxies, p.Clone())
+			for i, _ := range proxies {
+				if _, ok := okMap[proxies[i].Identifier()]; ok {
+					//cproxies = append(cproxies, p) // 没必要Clone因为range时就复制了一份
+					cproxies = append(cproxies, proxies[i]) // 使用对GC非常不友好的指针看看会不会有事
 				}
 			}
 			return

pkg/proxy/base.go

@@ -9,16 +9,13 @@ import (
 type Base struct {
 	Name    string `yaml:"name" json:"name" gorm:"index"`
 	Server  string `yaml:"server" json:"server" gorm:"index"`
-	Port    int    `yaml:"port" json:"port" gorm:"index"`
 	Type    string `yaml:"type" json:"type" gorm:"index"`
-	UDP     bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
 	Country string `yaml:"country,omitempty" json:"country,omitempty" gorm:"index"`
-	// 这个单词的原作者拼写是错误的，但我不想改了，我也没有早点发现这件事，在写where查询老写错，非常的无奈
-	Useable bool `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"`
+	Port    int    `yaml:"port" json:"port" gorm:"index"`
+	UDP     bool   `yaml:"udp,omitempty" json:"udp,omitempty"`
+	Useable bool   `yaml:"useable,omitempty" json:"useable,omitempty" gorm:"index"`
 }
 
-// Note: Go只有值传递，必需传入指针才能改变传入的结构体
-
 // TypeName() Get specific proxy type
 func (b *Base) TypeName() string {
 	if b.Type == "" {

pkg/proxy/convert.go

@@ -26,7 +26,7 @@ func Convert2SSR(p Proxy) (ssr *ShadowsocksR, err error) {
 			Cipher:   ss.Cipher,
 			Protocol: "origin",
 			Obfs:     "plain",
-			Group:    "proxy.tgbot.co",
+			Group:    "",
 		}, nil
 	}
 	return nil, ErrorTypeCanNotConvert

pkg/proxy/shadowsocks.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"math/rand"
 	"net"
 	"net/url"
 	"regexp"
@@ -143,7 +142,7 @@ func ParseSSLink(link string) (*Shadowsocks, error) {
 
 	return &Shadowsocks{
 		Base: Base{
-			Name:   strconv.Itoa(rand.Int()),
+			Name:   "",
 			Server: server,
 			Port:   port,
 			Type:   "ss",

pkg/proxy/shadowsocksr.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"math/rand"
 	"net"
 	"net/url"
 	"regexp"
@@ -70,7 +69,7 @@ func (ssr ShadowsocksR) Link() (link string) {
 	query := url.Values{}
 	query.Add("obfsparam", tool.Base64EncodeString(ssr.ObfsParam, true))
 	query.Add("protoparam", tool.Base64EncodeString(ssr.ProtocolParam, true))
-	query.Add("remarks", tool.Base64EncodeString(ssr.Name, true))
+	//query.Add("remarks", tool.Base64EncodeString(ssr.Name, true))
 	query.Add("group", tool.Base64EncodeString("proxypoolss.herokuapp.com", true))
 	payload = tool.Base64EncodeString(fmt.Sprintf("%s/?%s", payload, query.Encode()), true)
 	return fmt.Sprintf("ssr://%s", payload)
@@ -113,18 +112,18 @@ func ParseSSRLink(link string) (*ShadowsocksR, error) {
 	moreInfo, _ := url.ParseQuery(infoPayload[1])
 
 	// remarks
-	remarks := moreInfo.Get("remarks")
-	remarks, err = tool.Base64DecodeString(remarks)
-	if err != nil {
-		remarks = ""
-		err = nil
-	}
-	if strings.ContainsAny(remarks, "\t\r\n ") {
-		remarks = strings.ReplaceAll(remarks, "\t", "")
-		remarks = strings.ReplaceAll(remarks, "\r", "")
-		remarks = strings.ReplaceAll(remarks, "\n", "")
-		remarks = strings.ReplaceAll(remarks, " ", "")
-	}
+	//remarks := moreInfo.Get("remarks")
+	//remarks, err = tool.Base64DecodeString(remarks)
+	//if err != nil {
+	//	remarks = ""
+	//	err = nil
+	//}
+	//if strings.ContainsAny(remarks, "\t\r\n ") {
+	//	remarks = strings.ReplaceAll(remarks, "\t", "")
+	//	remarks = strings.ReplaceAll(remarks, "\r", "")
+	//	remarks = strings.ReplaceAll(remarks, "\n", "")
+	//	remarks = strings.ReplaceAll(remarks, " ", "")
+	//}
 
 	// protocol param
 	protocolParam, err := tool.Base64DecodeString(moreInfo.Get("protoparam"))
@@ -150,15 +149,9 @@ func ParseSSRLink(link string) (*ShadowsocksR, error) {
 		obfs = strings.ReplaceAll(obfs, "_compatible", "")
 	}
 
-	//group, err := tool.Base64DecodeString(moreInfo.Get("group"))
-	//if err != nil {
-	//	group = ""
-	//}
-	group := ""
-
 	return &ShadowsocksR{
 		Base: Base{
-			Name:   remarks + "_" + strconv.Itoa(rand.Int()),
+			Name:   "",
 			Server: server,
 			Port:   port,
 			Type:   "ssr",
@@ -169,7 +162,7 @@ func ParseSSRLink(link string) (*ShadowsocksR, error) {
 		ProtocolParam: protocolParam,
 		Obfs:          obfs,
 		ObfsParam:     obfsParam,
-		Group:         group,
+		Group:         "",
 	}, nil
 }
 

pkg/proxy/trojan.go

@@ -3,7 +3,6 @@ package proxy
 import (
 	"encoding/json"
 	"errors"
-	"math/rand"
 	"net"
 	"net/url"
 	"regexp"
@@ -138,15 +137,15 @@ func ParseTrojanLink(link string) (*Trojan, error) {
 
 	return &Trojan{
 		Base: Base{
-			Name:   strconv.Itoa(rand.Int()),
+			Name:   "",
 			Server: server,
 			Port:   port,
 			Type:   "trojan",
 		},
 		Password:       password,
 		ALPN:           alpn,
-		UDP:            true,
 		SNI:            host,
+		UDP:            true,
 		SkipCertVerify: true,
 	}, nil
 }

pkg/proxy/vmess.go

@@ -25,14 +25,14 @@ type Vmess struct {
 	UUID           string            `yaml:"uuid" json:"uuid"`
 	AlterID        int               `yaml:"alterId" json:"alterId"`
 	Cipher         string            `yaml:"cipher" json:"cipher"`
-	TLS            bool              `yaml:"tls,omitempty" json:"tls,omitempty"`
 	Network        string            `yaml:"network,omitempty" json:"network,omitempty"`
-	HTTPOpts       HTTPOptions       `yaml:"http-opts,omitempty" json:"http-opts,omitempty"`
-	HTTP2Opts      HTTP2Options      `yaml:"h2-opts,omitempty" json:"h2-opts,omitempty"`
 	WSPath         string            `yaml:"ws-path,omitempty" json:"ws-path,omitempty"`
+	ServerName     string            `yaml:"servername,omitempty" json:"servername,omitempty"`
 	WSHeaders      map[string]string `yaml:"ws-headers,omitempty" json:"ws-headers,omitempty"`
+	HTTPOpts       HTTPOptions       `yaml:"http-opts,omitempty" json:"http-opts,omitempty"`
+	HTTP2Opts      HTTP2Options      `yaml:"h2-opts,omitempty" json:"h2-opts,omitempty"`
+	TLS            bool              `yaml:"tls,omitempty" json:"tls,omitempty"`
 	SkipCertVerify bool              `yaml:"skip-cert-verify,omitempty" json:"skip-cert-verify,omitempty"`
-	ServerName     string            `yaml:"servername,omitempty" json:"servername,omitempty"`
 }
 
 type HTTPOptions struct {
@@ -314,7 +314,7 @@ func ParseVmessLink(link string) (*Vmess, error) {
 
 		return &Vmess{
 			Base: Base{
-				Name:   vmessJson.Ps + "_" + strconv.Itoa(rand.Int()),
+				Name:   "",
 				Server: vmessJson.Add,
 				Port:   vmessJson.Port,
 				Type:   "vmess",
@@ -323,14 +323,14 @@ func ParseVmessLink(link string) (*Vmess, error) {
 			UUID:           vmessJson.Id,
 			AlterID:        alterId,
 			Cipher:         "auto",
-			TLS:            tls,
 			Network:        vmessJson.Net,
 			HTTPOpts:       httpOpt,
 			HTTP2Opts:      h2Opt,
 			WSPath:         vmessJson.Path,
 			WSHeaders:      wsHeaders,
-			SkipCertVerify: true,
 			ServerName:     vmessJson.Host,
+			TLS:            tls,
+			SkipCertVerify: true,
 		}, nil
 	}
 }
@@ -385,6 +385,8 @@ func mapStrInter2VmessLinkJson(jsn map[string]interface{}) (vmessLinkJson, error
 				default:
 					vmessVal.Field(i).SetInt(443)
 				}
+			} else if strings.ToLower(tag[0]) == "ps" {
+				continue
 			} else { // set string in other fields
 				switch jsnVal.(type) {
 				case string: