add cf script parser

go.mod

@@ -31,6 +31,7 @@ require (
 	github.com/oschwald/geoip2-golang v1.4.0
 	github.com/oschwald/maxminddb-golang v1.7.0 // indirect
 	github.com/patrickmn/go-cache v2.1.0+incompatible
+	github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac
 	github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect
 	github.com/temoto/robotstxt v1.1.1 // indirect
 	golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a // indirect
@@ -39,6 +40,7 @@ require (
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.25.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gorm.io/driver/postgres v1.0.0
 	gorm.io/gorm v1.20.0
 )

go.sum

@@ -276,6 +276,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rafaeljusto/redigomock v0.0.0-20190202135759-257e089e14a1/go.mod h1:JaY6n2sDr+z2WTsXkOmNRUfDy6FN0L6Nk7x06ndm4tY=
 github.com/rcrowley/go-metrics v0.0.0-20160613154715-cfa5a85e9f0a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
+github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac h1:kYPjbEN6YPYWWHI6ky1J813KzIq/8+Wg4TO4xU7A/KU=
+github.com/robertkrimen/otto v0.0.0-20200922221731-ef014fd054ac/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY=
 github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62 h1:pyecQtsPmlkCsMkYhT5iZ+sUXuwee+OvfuJjinEA3ko=
 github.com/robfig/go-cache v0.0.0-20130306151617-9fc39e0dbf62/go.mod h1:65XQgovT59RWatovFwnwocoUxiI/eENTnOY5GK3STuY=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
@@ -500,6 +502,8 @@ gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/R
 gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s=
 gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/sourcemap.v1 v1.0.5 h1:inv58fC9f9J3TK2Y2R1NPntXEn3/wjWHkonhIUODNTI=
+gopkg.in/sourcemap.v1 v1.0.5/go.mod h1:2RlvNNSMglmRrcvhfuzp4hQHwOtjxlbjX7UPY/GXb78=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/getter/web_fanqiangdang.go

@@ -41,7 +41,7 @@ func (w *WebFanqiangdang) Get() proxy.ProxyList {
 	w.results = make(proxy.ProxyList, 0)
 	w.c.OnHTML("td.t_f", func(e *colly.HTMLElement) {
 		if strings.Contains(e.Text, "data-cfemail") {
-			mail := tool.CFDecode(tool.GetCFPayload(e.Text))
+			mail := tool.CFEmailDecode(tool.GetCFEmailPayload(e.Text))
 			var re = regexp.MustCompile(`<a.*?href="/cdn-cgi.*?".*?>(.+?)</a>`)
 			e.Text = re.ReplaceAllString(e.Text, mail)
 		}

pkg/tool/cfdecode.go

@@ -2,11 +2,13 @@ package tool
 
 import (
 	"bytes"
+	"github.com/robertkrimen/otto"
+	"regexp"
 	"strconv"
 	"strings"
 )
 
-func GetCFPayload(str string) string {
+func GetCFEmailPayload(str string) string {
 	s := strings.Split(str, "data-cfemail=")
 	if len(s) > 1 {
 		s = strings.Split(s[1], "\"")
@@ -16,7 +18,8 @@ func GetCFPayload(str string) string {
 	return ""
 }
 
-func CFDecode(a string) (s string) {
+// Remove cloud flare email protection
+func CFEmailDecode(a string) (s string) {
 	if a == "" {
 		return
 	}
@@ -28,3 +31,67 @@ func CFDecode(a string) (s string) {
 	}
 	return e.String()
 }
+
+// Get result var of a js script
+func ScriptGet(js string, varname string) string {
+	vm := otto.New()
+	_, err := vm.Run(js)
+	if err != nil {
+		return ""
+	}
+	if value, err := vm.Get(varname); err == nil {
+		if v, err := value.ToString(); err == nil {
+			return v
+		}
+	}
+	return ""
+}
+
+// replace location with varname and remove window
+func ScriptReplace(js string, varname string) string {
+	strs := strings.Split(js, ";")
+	varWindow := ""
+	varLocation := ""
+	bound := len(strs)
+
+	if len(js) < 2 {
+		return js
+	}
+	for i, _ := range strs {
+		//replace location
+		if strings.Contains(strs[i], "location") {
+			strarr := strings.Split(strs[i], " = ")
+			if len(strarr) == 2 {
+				varLocation = strarr[0]
+				strs[i] = ""
+			} else {
+				re3, err := regexp.Compile("location.*?[]]") // location[_jzvXT]
+				if err == nil {
+					strs[i] = re3.ReplaceAllLiteralString(strs[i], varname)
+				}
+			}
+		}
+		if varLocation != "" && strings.Contains(strs[i], varLocation) {
+			re3, err := regexp.Compile(varLocation + ".*?[]]") // _LoKlO[_jzvXT]
+			if err == nil {
+				strs[i] = re3.ReplaceAllLiteralString(strs[i], varname)
+			}
+		}
+		// remove window
+		if strings.Contains(strs[i], "window") {
+			varWindow = strings.Split(strs[i], " = window")[0]
+			strs[i] = ""
+		}
+	}
+
+	if varWindow != "" {
+		for i, _ := range strs {
+			if strings.Contains(strs[i], varWindow) {
+				bound = i
+				break
+			}
+		}
+	}
+	js = strings.Join(strs[:bound], ";")
+	return js
+}