use config file

madxzysy · Aug 12, 2020 · 21f55ac · 21f55ac
1 parent c87b587
commit 21f55ac
Show file tree

Hide file tree

Showing 16 changed files with 691 additions and 108 deletions.
diff --git a/app/getter.go b/app/getter.go
@@ -0,0 +1,21 @@
+package app
+
+import (
+	"fmt"
+
+	"github.com/zu1k/proxypool/config"
+	"github.com/zu1k/proxypool/getter"
+)
+
+var Getters = make([]getter.Getter, 0)
+
+func InitGetters(sources []config.Source) {
+	for _, source := range sources {
+		g := getter.NewGetter(source.Type, source.Options)
+		if g != nil {
+			Getters = append(Getters, g)
+			fmt.Println("init getter:", source.Type, source.Options)
+		}
+	}
+	fmt.Println("Getter count:", len(Getters))
+}
diff --git a/app/task.go b/app/task.go
@@ -10,43 +10,14 @@ import (
 
 	"github.com/zu1k/proxypool/app/cache"
 
-	"github.com/zu1k/proxypool/getter"
 	"github.com/zu1k/proxypool/proxy"
 )
 
 func Crawl() {
 	proxies := make([]proxy.Proxy, 0)
-
-	// tg上各种节点分享频道
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrList", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/SSRSUB", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/FreeSSRNode", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrlists", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrshares", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/V2List", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/ssrtool", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/vmessr", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/FreeSSR666", 200).Get()...)
-	proxies = append(proxies, getter.NewTGChannelGetter("https://t.me/s/fanqiang666", 200).Get()...)
-
-	// 各种网站上公开的
-	proxies = append(proxies, getter.WebFreessrXyz{}.Get()...)
-	proxies = append(proxies, getter.WebLucnOrg{}.Get()...)
-
-	// 从web页面模糊获取
-	proxies = append(proxies, getter.NewWebFuzz("https://zfjvpn.gitbook.io/").Get()...)
-	proxies = append(proxies, getter.NewWebFuzz("https://www.freefq.com/d/file/free-ssr/20200811/1f3e9d0d0064f662457062712dcf1b66.txt").Get()...)
-	proxies = append(proxies, getter.NewWebFuzz("https://merlinblog.xyz/wiki/freess.html").Get()...)
-	// 翻墙党
-	proxies = append(proxies, getter.NewWebFanqiangdangGetter("https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0", 200).Get()...)
-	proxies = append(proxies, getter.NewWebFanqiangdangGetter("https://fanqiangdang.com/forum.php?mod=rss&fid=2&auth=0", 200).Get()...)
-	proxies = append(proxies, getter.NewWebFanqiangdangGetter("https://fanqiangdang.com/forum.php?mod=rss&fid=36&auth=0", 200).Get()...)
-
-	// 订阅链接
-	proxies = append(proxies, getter.NewSubscribe("https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray").Get()...)
-	proxies = append(proxies, getter.NewSubscribe("https://raw.githubusercontent.com/ssrsub/ssr/master/ssrsub").Get()...)
-	proxies = append(proxies, getter.NewSubscribe("https://raw.githubusercontent.com/ssrsub/ssr/master/ss-sub").Get()...)
-
+	for _, g := range Getters {
+		proxies = append(proxies, g.Get()...)
+	}
 	proxies = append(proxies, cache.GetProxies()...)
 	proxies = proxy.Deduplication(proxies)
 
@@ -60,38 +31,11 @@ func Crawl() {
 }
 
 func CrawlGo() {
-	wg := sync.WaitGroup{}
+	wg := &sync.WaitGroup{}
 	var pc = make(chan proxy.Proxy)
-	// tg上各种节点分享频道
-	go getter.NewTGChannelGetter("https://t.me/s/ssrList", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/SSRSUB", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/FreeSSRNode", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/ssrlists", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/ssrshares", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/V2List", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/ssrtool", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/vmessr", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/FreeSSR666", 200).Get2Chan(pc, &wg)
-	go getter.NewTGChannelGetter("https://t.me/s/fanqiang666", 200).Get2Chan(pc, &wg)
-
-	// 各种网站上公开的
-	go getter.WebFreessrXyz{}.Get2Chan(pc, &wg)
-	go getter.WebLucnOrg{}.Get2Chan(pc, &wg)
-
-	// 从web页面模糊获取
-	go getter.NewWebFuzz("https://zfjvpn.gitbook.io/").Get2Chan(pc, &wg)
-	go getter.NewWebFuzz("https://www.freefq.com/d/file/free-ssr/20200811/1f3e9d0d0064f662457062712dcf1b66.txt").Get2Chan(pc, &wg)
-	go getter.NewWebFuzz("https://merlinblog.xyz/wiki/freess.html").Get2Chan(pc, &wg)
-	// 翻墙党
-	go getter.NewWebFanqiangdangGetter("https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0", 200).Get2Chan(pc, &wg)
-	go getter.NewWebFanqiangdangGetter("https://fanqiangdang.com/forum.php?mod=rss&fid=2&auth=0", 200).Get2Chan(pc, &wg)
-	go getter.NewWebFanqiangdangGetter("https://fanqiangdang.com/forum.php?mod=rss&fid=36&auth=0", 200).Get2Chan(pc, &wg)
-
-	// 订阅链接
-	go getter.NewSubscribe("https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray").Get2Chan(pc, &wg)
-	go getter.NewSubscribe("https://raw.githubusercontent.com/ssrsub/ssr/master/ssrsub").Get2Chan(pc, &wg)
-	go getter.NewSubscribe("https://raw.githubusercontent.com/ssrsub/ssr/master/ss-sub").Get2Chan(pc, &wg)
-
+	for _, g := range Getters {
+		go g.Get2Chan(pc, wg)
+	}
 	proxies := cache.GetProxies()
 	go func() {
 		wg.Wait()

diff --git a/config/config.go b/config/config.go
@@ -0,0 +1,50 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	"github.com/ghodss/yaml"
+	"github.com/zu1k/proxypool/tool"
+)
+
+type Source struct {
+	Type    string       `json:"type" yaml:"type"`
+	Options tool.Options `json:"options" yaml:"options"`
+}
+
+type Config struct {
+	Sources []Source `json:"sources" yaml:"sources"`
+}
+
+var SourceConfig = Config{}
+
+func Parse(path string) (*Config, error) {
+	fileData, err := readFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	err = yaml.Unmarshal(fileData, &SourceConfig)
+	if err != nil {
+		return nil, err
+	}
+	return &SourceConfig, nil
+}
+
+func readFile(path string) ([]byte, error) {
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		return nil, err
+	}
+	data, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(data) == 0 {
+		return nil, fmt.Errorf("Configuration file %s is empty", path)
+	}
+
+	return data, err
+}
diff --git a/example/source.yaml b/example/source.yaml
@@ -0,0 +1,24 @@
+sources:
+  - type: subscribe
+    options:
+      url: https://raw.githubusercontent.com/ssrsub/ssr/master/v2ray
+
+  - type: webfuzz
+    options:
+      url: https://merlinblog.xyz/wiki/freess.html
+
+  - type: tgchannel
+    options:
+      channel: ssrList
+      num: 200
+
+  - type: web-fanqiangdang
+    options:
+      url: https://fanqiangdang.com/forum.php?mod=rss&fid=50&auth=0
+      num: 200
+
+  - type: web-freessrxyz
+    options:
+
+  - type: web-lucnorg
+    options:
diff --git a/getter/base.go b/getter/base.go
@@ -5,11 +5,28 @@ import (
 	"sync"
 
 	"github.com/zu1k/proxypool/proxy"
+	"github.com/zu1k/proxypool/tool"
 )
 
 type Getter interface {
 	Get() []proxy.Proxy
-	Get2Chan(pc chan proxy.Proxy, wg sync.WaitGroup)
+	Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup)
+}
+
+type creator func(options tool.Options) Getter
+
+var creatorMap = make(map[string]creator)
+
+func Register(sourceType string, c creator) {
+	creatorMap[sourceType] = c
+}
+
+func NewGetter(sourceType string, options tool.Options) Getter {
+	c, ok := creatorMap[sourceType]
+	if ok {
+		return c(options)
+	}
+	return nil
 }
 
 func String2Proxy(link string) proxy.Proxy {

diff --git a/getter/subscribe.go b/getter/subscribe.go
@@ -10,11 +10,15 @@ import (
 	"github.com/zu1k/proxypool/tool"
 )
 
+func init() {
+	Register("subscribe", NewSubscribe)
+}
+
 type Subscribe struct {
 	Url string
 }
 
-func (s Subscribe) Get() []proxy.Proxy {
+func (s *Subscribe) Get() []proxy.Proxy {
 	resp, err := http.Get(s.Url)
 	if err != nil {
 		return nil
@@ -35,7 +39,7 @@ func (s Subscribe) Get() []proxy.Proxy {
 	return StringArray2ProxyArray(nodes)
 }
 
-func (s Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	wg.Add(1)
 	nodes := s.Get()
 	for _, node := range nodes {
@@ -44,8 +48,12 @@ func (s Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	wg.Done()
 }
 
-func NewSubscribe(url string) *Subscribe {
-	return &Subscribe{
-		Url: url,
+func NewSubscribe(options tool.Options) Getter {
+	url, found := options["url"]
+	if found {
+		return &Subscribe{
+			Url: url.(string),
+		}
 	}
+	return nil
 }
diff --git a/getter/tgchannel.go b/getter/tgchannel.go
@@ -4,52 +4,63 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/zu1k/proxypool/tool"
+
 	"github.com/gocolly/colly"
 	"github.com/zu1k/proxypool/proxy"
 )
 
+func init() {
+	Register("tgchannel", NewTGChannelGetter)
+}
+
 type TGChannelGetter struct {
 	c         *colly.Collector
 	NumNeeded int
-	Results   []string
+	results   []string
 	Url       string
 }
 
-func NewTGChannelGetter(url string, numNeeded int) *TGChannelGetter {
-	if numNeeded <= 0 {
-		numNeeded = 200
+func NewTGChannelGetter(options tool.Options) Getter {
+	num, found := options["num"]
+	if !found || int(num.(float64)) <= 0 {
+		num = 200
 	}
-	return &TGChannelGetter{
-		c:         colly.NewCollector(),
-		NumNeeded: numNeeded,
-		Results:   make([]string, 0),
-		Url:       url,
+	url, found := options["channel"]
+	if found {
+		return &TGChannelGetter{
+			c:         colly.NewCollector(),
+			NumNeeded: int(num.(float64)),
+			Url:       "https://t.me/s/" + url.(string),
+		}
 	}
+	return nil
 }
 
-func (g TGChannelGetter) Get() []proxy.Proxy {
+func (g *TGChannelGetter) Get() []proxy.Proxy {
+	g.results = make([]string, 0)
 	// 找到所有的文字消息
 	g.c.OnHTML("div.tgme_widget_message_text", func(e *colly.HTMLElement) {
-		g.Results = append(g.Results, GrepLinksFromString(e.Text)...)
+		g.results = append(g.results, GrepLinksFromString(e.Text)...)
 	})
 
 	// 找到之前消息页面的链接，加入访问队列
 	g.c.OnHTML("link[rel=prev]", func(e *colly.HTMLElement) {
-		if len(g.Results) < g.NumNeeded {
+		if len(g.results) < g.NumNeeded {
 			_ = e.Request.Visit(e.Attr("href"))
 		}
 	})
 
-	g.Results = make([]string, 0)
+	g.results = make([]string, 0)
 	err := g.c.Visit(g.Url)
 	if err != nil {
 		_ = fmt.Errorf("%s", err.Error())
 	}
 
-	return StringArray2ProxyArray(g.Results)
+	return StringArray2ProxyArray(g.results)
 }
 
-func (g TGChannelGetter) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	wg.Add(1)
 	nodes := g.Get()
 	for _, node := range nodes {