Merge pull request xlvector#15 from GapYearWanted/master

add url in Cmd.Output;remove time sleep
chaopengio · May 31, 2016 · d40ed88 · d40ed88
2 parents 6d1a76c + 2f555ff
commit d40ed88
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 2 deletions.
diff --git a/cmd/cmd.go b/cmd/cmd.go
@@ -48,4 +48,5 @@ type Output struct {
 	NeedParam string `json:"need_param"`
 	Id        string `json:"id"`
 	Data      string `json:"data"`
+	Url 	  string `json:"url"`
 }
diff --git a/task/task_cmd.go b/task/task_cmd.go
@@ -12,7 +12,7 @@ import (
 	hproxy "github.com/xlvector/higgs/proxy"
 	"github.com/xlvector/higgs/util"
 	"io/ioutil"
-	"math/rand"
+	_"math/rand"
 	"net/url"
 	"os"
 	"runtime/debug"
@@ -33,6 +33,7 @@ type TaskCmd struct {
 	userId       string
 	passWord     string
 	path         string
+	url	     string
 	message      chan *cmd.Output
 	input        chan map[string]string
 	args         map[string]string
@@ -143,6 +144,7 @@ func (s *TaskCmdFactory) createCommandWithPrivateKey(params url.Values, task *Ta
 		RedisHost:    config.Instance.Redis.Host,
 		RedisTimeout: time.Duration(config.Instance.Redis.Timeout),
 	},   s.proxyManager)
+	dlog.Println(ret.downloader.Client)
 
 	dlog.Warn("output folder: %s", ret.downloader.OutputFolder)
 	ret.downloader.Context.Set("_id", ret.GetId())
@@ -283,7 +285,7 @@ func (p *TaskCmd) run() {
 		}
 
 		step := p.task.Steps[c]
-		time.Sleep(time.Duration(rand.Int63n(300)) * time.Millisecond)
+		//time.Sleep(time.Duration(rand.Int63n(300)) * time.Millisecond)
 
 		if len(step.NeedParam) > 0 {
 			tks := strings.Split(step.NeedParam, ",")
@@ -296,6 +298,9 @@ func (p *TaskCmd) run() {
 						val = util.DecodePassword(val, p.privateKey)
 					}
 					p.downloader.Context.Set(tk, val)
+				} else {
+					url,_ := p.downloader.Context.Get(tk)
+					p.url = url.(string)
 				}
 			}
 		}
@@ -324,6 +329,7 @@ func (p *TaskCmd) run() {
 				Status: cmd.WRONG_RESPONSE,
 				Id:	p.GetArgsValue("id"),
 				Data:	p.downloader.Context.Parse(step.Message["data"]),
+				Url:	p.url,
 			}
 
 			p.message <- msg
@@ -340,6 +346,7 @@ func (p *TaskCmd) run() {
 					Status: cmd.TMPL_BLOCK,
 					Id:	p.GetArgsValue("id"),
 					Data:	data,
+					Url: 	p.url,
 				}
 				dlog.Println(data)
 				p.message <- msg
@@ -353,6 +360,7 @@ func (p *TaskCmd) run() {
 					Status: step.Message["status"],
 					Id:     p.GetArgsValue("id"),
 					Data:   data,
+					Url:	p.url,
 				}
 
 				if needParam, ok := step.Message["need_param"]; ok {
@@ -377,6 +385,7 @@ func (p *TaskCmd) run() {
 					Id:        p.GetArgsValue("id"),
 					NeedParam: action.Message["need_param"],
 					Data:      actionInfo,
+					Url:	   p.url,
 				}
 
 				p.message <- msg
@@ -405,6 +414,7 @@ func (p *TaskCmd) run() {
 						Status: cmd.FAIL,
 						Id:     p.GetArgsValue("id"),
 						Data:   actionInfo,
+						Url:	p.url,
 					}
 					p.message <- msg
 					dlog.Warn("%s Status:%s", p.GetId(), "retry fail "+step.Page)
@@ -450,6 +460,7 @@ func (p *TaskCmd) run() {
 		Status: cmd.FINISH_FETCH_DATA,
 		Id:     p.GetArgsValue("id"),
 		Data:   p.downloader.ExtractorResultString(),
+		Url:	p.url,
 	}
 
 	p.message <- message