rename a function

Robbie6666 · Nov 27, 2020 · 566520f · 566520f
1 parent 2e28eaf
commit 566520f
Show file tree

Hide file tree

Showing 8 changed files with 57 additions and 9 deletions.
diff --git a/internal/app/task.go b/internal/app/task.go
@@ -22,7 +22,7 @@ func CrawlGo() {
 	for _, g := range Getters {
 		wg.Add(1)
 		// 并发执行抓取node并存到pc
-		go g.Get2Chan(pc, wg)
+		go g.Get2ChanWG(pc, wg)
 	}
 	proxies := cache.GetProxies("allproxies")
 	dbProxies := database.GetAllProxies()
@@ -43,6 +43,7 @@ func CrawlGo() {
 		wg.Wait()
 		close(pc)
 	}() // Note: 为何并发？可以一边抓取一边读取而非抓完再读
+	// for 用于阻塞goroutine
 	for p := range pc { // Note: pc关闭后不能发送数据可以读取剩余数据
 		if p != nil {
 			proxies = proxies.UniqAppendProxy(p)

diff --git a/pkg/getter/base.go b/pkg/getter/base.go
@@ -11,7 +11,8 @@ import (
 // functions for getters
 type Getter interface {
 	Get() proxy.ProxyList
-	Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup)
+	Get2Chan(pc chan proxy.Proxy)
+	Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup)
 }
 
 // function type that creates getters

diff --git a/pkg/getter/subscribe.go b/pkg/getter/subscribe.go
@@ -43,7 +43,7 @@ func (s *Subscribe) Get() proxy.ProxyList {
 }
 
 // Get2Chan() of Subscribe is to implement Getter interface. It gets proxies and send proxy to channel one by one
-func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (s *Subscribe) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := s.Get()
 	log.Printf("STATISTIC: Subscribe\tcount=%d\turl=%s\n", len(nodes), s.Url)
@@ -52,6 +52,14 @@ func (s *Subscribe) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	}
 }
 
+func (s *Subscribe) Get2Chan(pc chan proxy.Proxy) {
+	nodes := s.Get()
+	log.Printf("STATISTIC: Subscribe\tcount=%d\turl=%s\n", len(nodes), s.Url)
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
 func NewSubscribe(options tool.Options) (getter Getter, err error) {
 	urlInterface, found := options["url"]
 	if found {

diff --git a/pkg/getter/tgchannel.go b/pkg/getter/tgchannel.go
@@ -102,11 +102,18 @@ func (g *TGChannelGetter) Get() proxy.ProxyList {
 	return result
 }
 
-func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (g *TGChannelGetter) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := g.Get()
 	log.Printf("STATISTIC: TGChannel\tcount=%d\turl=%s\n", len(nodes), g.Url)
 	for _, node := range nodes {
 		pc <- node
 	}
 }
+func (g *TGChannelGetter) Get2Chan(pc chan proxy.Proxy) {
+	nodes := g.Get()
+	log.Printf("STATISTIC: TGChannel\tcount=%d\turl=%s\n", len(nodes), g.Url)
+	for _, node := range nodes {
+		pc <- node
+	}
+}
diff --git a/pkg/getter/web_fanqiangdang.go b/pkg/getter/web_fanqiangdang.go
@@ -79,11 +79,19 @@ func (w *WebFanqiangdang) Get() proxy.ProxyList {
 	return w.results
 }
 
-func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (w *WebFanqiangdang) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
 	log.Printf("STATISTIC: Fanqiangdang\tcount=%d\turl=%s\n", len(nodes), w.Url)
 	for _, node := range nodes {
 		pc <- node
 	}
 }
+
+func (w *WebFanqiangdang) Get2Chan(pc chan proxy.Proxy) {
+	nodes := w.Get()
+	log.Printf("STATISTIC: Fanqiangdang\tcount=%d\turl=%s\n", len(nodes), w.Url)
+	for _, node := range nodes {
+		pc <- node
+	}
+}
diff --git a/pkg/getter/web_free_ssr_xyz.go b/pkg/getter/web_free_ssr_xyz.go
@@ -32,7 +32,7 @@ func (w *WebFreessrXyz) Get() proxy.ProxyList {
 	return results
 }
 
-func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (w *WebFreessrXyz) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
 	log.Printf("STATISTIC: FreeSSRxyz\tcount=%d\turl=%s\n", len(nodes), "api.free-ssr.xyz")
@@ -41,6 +41,14 @@ func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	}
 }
 
+func (w *WebFreessrXyz) Get2Chan(pc chan proxy.Proxy) {
+	nodes := w.Get()
+	log.Printf("STATISTIC: FreeSSRxyz\tcount=%d\turl=%s\n", len(nodes), "api.free-ssr.xyz")
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
 func freessrxyzFetch(link string) proxy.ProxyList {
 	resp, err := tool.GetHttpClient().Get(link)
 	if err != nil {

diff --git a/pkg/getter/web_fuzz.go b/pkg/getter/web_fuzz.go
@@ -31,11 +31,10 @@ func (w *WebFuzz) Get() proxy.ProxyList {
 	if err != nil {
 		return nil
 	}
-
 	return FuzzParseProxyFromString(string(body))
 }
 
-func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (w *WebFuzz) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
 	log.Printf("STATISTIC: WebFuzz\tcount=%d\turl=%s\n", len(nodes), w.Url)
@@ -44,6 +43,14 @@ func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	}
 }
 
+func (w *WebFuzz) Get2Chan(pc chan proxy.Proxy) {
+	nodes := w.Get()
+	log.Printf("STATISTIC: WebFuzz\tcount=%d\turl=%s\n", len(nodes), w.Url)
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
 func NewWebFuzzGetter(options tool.Options) (getter Getter, err error) {
 	urlInterface, found := options["url"]
 	if found {

diff --git a/pkg/getter/web_fuzz_sub.go b/pkg/getter/web_fuzz_sub.go
@@ -37,7 +37,7 @@ func (w *WebFuzzSub) Get() proxy.ProxyList {
 	return result
 }
 
-func (w *WebFuzzSub) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
+func (w *WebFuzzSub) Get2ChanWG(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	defer wg.Done()
 	nodes := w.Get()
 	log.Printf("STATISTIC: WebFuzzSub\tcount=%d\turl=%s\n", len(nodes), w.Url)
@@ -46,6 +46,14 @@ func (w *WebFuzzSub) Get2Chan(pc chan proxy.Proxy, wg *sync.WaitGroup) {
 	}
 }
 
+func (w *WebFuzzSub) Get2Chan(pc chan proxy.Proxy) {
+	nodes := w.Get()
+	log.Printf("STATISTIC: WebFuzzSub\tcount=%d\turl=%s\n", len(nodes), w.Url)
+	for _, node := range nodes {
+		pc <- node
+	}
+}
+
 func NewWebFuzzSubGetter(options tool.Options) (getter Getter, err error) {
 	urlInterface, found := options["url"]
 	if found {