fix linkfinder print and visit

thr3athunt3r · Jun 17, 2021 · 2070c11 · 2070c11
1 parent 028c070
commit 2070c11
Showing 1 changed file with 45 additions and 30 deletions.
diff --git a/core/crawler.go b/core/crawler.go
@@ -568,10 +568,21 @@ func (crawler *Crawler) setupLinkFinder() {
 			return
 		}
 
-		u := response.Request.URL.String()
-
 		respStr := string(response.Body)
 		if len(crawler.filterLength_slice) == 0 || !contains(crawler.filterLength_slice,len(respStr)) {
+
+			// Verify which link is working
+			u := response.Request.URL.String()
+			outputFormat := fmt.Sprintf("[url] - [code-%d] - %s", response.StatusCode, u)
+
+			if crawler.length {
+				outputFormat = fmt.Sprintf("[url] - [code-%d] - [len_%d] - %s", response.StatusCode, len(respStr), u)
+			}
+			fmt.Println(outputFormat)
+
+			if crawler.Output != nil {
+				crawler.Output.WriteToFile(outputFormat)
+			}
 
 			crawler.findAWSS3(respStr)
 			crawler.findSubdomains(respStr)
@@ -620,40 +631,44 @@ func (crawler *Crawler) setupLinkFinder() {
 					continue
 				}
 
-
-
-				if crawler.JsonOutput {
-					sout := SpiderOutput{
-						Input:      crawler.Input,
-						Source:     response.Request.URL.String(),
-						OutputType: "linkfinder",
-						Output:     rebuildURL,
+				fileExt := GetExtType(rebuildURL)
+				if fileExt == ".js" || fileExt == ".xml" || fileExt == ".json" {
+					_ = crawler.LinkFinderCollector.Visit(rebuildURL)
+				}else if !crawler.urlSet.Duplicate(rebuildURL){
+
+					if crawler.JsonOutput {
+						sout := SpiderOutput{
+							Input:      crawler.Input,
+							Source:     response.Request.URL.String(),
+							OutputType: "linkfinder",
+							Output:     rebuildURL,
+						}
+						if data, err := jsoniter.MarshalToString(sout); err == nil {
+							outputFormat = data
+						}
+					} else if !crawler.Quiet {
+						outputFormat = fmt.Sprintf("[linkfinder] - %s", rebuildURL)
 					}
-					if data, err := jsoniter.MarshalToString(sout); err == nil {
-						outputFormat = data
-					}
-				} else if !crawler.Quiet {
-					outputFormat = fmt.Sprintf("[linkfinder] - %s", rebuildURL)
-				}
 
-				fmt.Println(outputFormat)
+					fmt.Println(outputFormat)
 
-				if crawler.Output != nil {
-					crawler.Output.WriteToFile(outputFormat)
-				}
+					if crawler.Output != nil {
+						crawler.Output.WriteToFile(outputFormat)
+					}
 
-				// Try to request JS path
-				// Try to generate URLs with main site
+					// Try to request JS path
+					// Try to generate URLs with main site
 
-				if rebuildURL != "" {
-					_ = crawler.C.Visit(rebuildURL)
-				}
+					if rebuildURL != "" {
+						_ = crawler.C.Visit(rebuildURL)
+					}
 
-				// Try to generate URLs with the site where Javascript file host in (must be in main or sub domain)
-				if inScope {
-					urlWithJSHostIn := FixUrl(crawler.site, relPath)
-					if urlWithJSHostIn != "" {
-						_ = crawler.C.Visit(urlWithJSHostIn)
+					// Try to generate URLs with the site where Javascript file host in (must be in main or sub domain)
+					if inScope {
+						urlWithJSHostIn := FixUrl(crawler.site, relPath)
+						if urlWithJSHostIn != "" {
+							_ = crawler.C.Visit(urlWithJSHostIn)
+						}
 					}
 				}
 			}