diff --git a/README.md b/README.md index 35a8a7547..9322f7ca1 100644 --- a/README.md +++ b/README.md @@ -27,9 +27,9 @@ The OWASP Amass Project has developed a tool to help information security profes **Information Gathering Techniques Used:** * **DNS:** Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request) -* **Scraping:** Ask, Baidu, Bing, CommonCrawl, DNSDumpster, DNSTable, Dogpile, Exalead, Google, HackerOne, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ViewDNS, Yahoo +* **Scraping:** Ask, Baidu, Bing, DNSDumpster, DNSTable, Dogpile, Exalead, Google, HackerOne, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ViewDNS, Yahoo * **Certificates:** Active pulls (upon request), Censys, CertSpotter, Crtsh, Entrust, GoogleCT -* **APIs:** AlienVault, BinaryEdge, BufferOver, CIRCL, DNSDB, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse (CertDB & FindSubdomains), Sublist3rAPI, TeamCymru, ThreatCrowd, Twitter, Umbrella, URLScan, VirusTotal +* **APIs:** AlienVault, BinaryEdge, BufferOver, CIRCL, CommonCrawl, DNSDB, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse (CertDB & FindSubdomains), Sublist3rAPI, TeamCymru, ThreatCrowd, Twitter, Umbrella, URLScan, VirusTotal * **Web Archives:** ArchiveIt, ArchiveToday, Arquivo, LoCArchive, OpenUKArchive, UKGovArchive, Wayback ---- @@ -68,10 +68,13 @@ This project improves thanks to all the people who contribute: [![Follow on Twitter](https://img.shields.io/twitter/follow/sec_for_safety.svg?logo=twitter)](https://twitter.com/sec_for_safety) [![Follow on Twitter](https://img.shields.io/twitter/follow/ngkogkos.svg?logo=twitter)](https://github.com/ngkogkos) [![Follow on Twitter](https://img.shields.io/twitter/follow/Jhaddix.svg?logo=twitter)](https://twitter.com/Jhaddix) +[![Follow on Twitter](https://img.shields.io/twitter/follow/Vltraheaven.svg?logo=twitter)](https://twitter.com/Vltraheaven) ## Mentions +* [8 Free Tools to Be Showcased at Black Hat and DEF CON](https://www.darkreading.com/application-security/8-free-tools-to-be-showcased-at-black-hat-and-def-con/d/d-id/1335356?image_number=5) * [amass — Automated Attack Surface Mapping](https://danielmiessler.com/study/amass/) +* [Aquatone — A Tool for Domain Flyovers](https://github.com/michenriksen/aquatone) * [Collaborating with the Crowd – Recapping LevelUp 0X04](https://www.bugcrowd.com/blog/recapping_levelup_0x04/) * [Subdomain Enumeration: 2019 Workflow](https://0xpatrik.com/subdomain-enumeration-2019/) * [REMOTE CODE EXECUTION ! 😜 Recon Wins](https://medium.com/@vishnu0002/remote-code-execution-recon-wins-e9c1db79f3da) diff --git a/cmd/amass/db.go b/cmd/amass/db.go index 87a3b8f04..4f02d3046 100644 --- a/cmd/amass/db.go +++ b/cmd/amass/db.go @@ -16,6 +16,7 @@ import ( "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/graph" "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/fatih/color" ) @@ -25,7 +26,7 @@ const ( ) type dbArgs struct { - Domains utils.ParseStrings + Domains stringset.Set Enum int Options struct { DemoMode bool @@ -52,6 +53,8 @@ func runDBCommand(clArgs []string) { dbBuf := new(bytes.Buffer) dbCommand.SetOutput(dbBuf) + args.Domains = stringset.New() + dbCommand.BoolVar(&help1, "h", false, "Show the program usage message") dbCommand.BoolVar(&help2, "help", false, "Show the program usage message") dbCommand.Var(&args.Domains, "d", "Domain names separated by commas (can be used multiple times)") @@ -88,7 +91,7 @@ func runDBCommand(clArgs []string) { r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err) return } - args.Domains = utils.UniqueAppend(args.Domains, list...) + args.Domains.InsertMany(list...) } cfg := new(config.Config) @@ -98,7 +101,7 @@ func runDBCommand(clArgs []string) { args.Filepaths.Directory = cfg.Dir } if len(args.Domains) == 0 { - args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...) + args.Domains.InsertMany(cfg.Domains()...) } } else if args.Filepaths.ConfigFile != "" { r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err) @@ -122,7 +125,7 @@ func runDBCommand(clArgs []string) { } if args.Options.ListEnumerations { - listEnumerations(args.Domains, db) + listEnumerations(&args, db) return } @@ -172,7 +175,8 @@ func inputDataOperations(args *dbArgs, db graph.DataHandler) error { return nil } -func listEnumerations(domains []string, db graph.DataHandler) { +func listEnumerations(args *dbArgs, db graph.DataHandler) { + domains := args.Domains.Slice() enums := enumIDs(domains, db) if len(enums) == 0 { r.Fprintln(color.Error, "No enumerations found within the provided scope") @@ -198,11 +202,12 @@ func listEnumerations(domains []string, db graph.DataHandler) { } func showEnumeration(args *dbArgs, db graph.DataHandler) { + domains := args.Domains.Slice() var total int tags := make(map[string]int) asns := make(map[int]*utils.ASNSummaryData) - for _, out := range getEnumOutput(args.Enum, args.Domains, db) { - if len(args.Domains) > 0 && !domainNameInScope(out.Name, args.Domains) { + for _, out := range getEnumOutput(args.Enum, domains, db) { + if len(domains) > 0 && !domainNameInScope(out.Name, domains) { continue } diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go index f2275dd87..9eb05ef4d 100644 --- a/cmd/amass/enum.go +++ b/cmd/amass/enum.go @@ -23,6 +23,7 @@ import ( "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/enum" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/fatih/color" homedir "github.com/mitchellh/go-homedir" @@ -40,19 +41,19 @@ type enumArgs struct { Addresses utils.ParseIPs ASNs utils.ParseInts CIDRs utils.ParseCIDRs - AltWordList []string - AltWordListMask utils.ParseStrings - BruteWordList []string - BruteWordListMask utils.ParseStrings - Blacklist utils.ParseStrings - Domains utils.ParseStrings - Excluded utils.ParseStrings - Included utils.ParseStrings + AltWordList stringset.Set + AltWordListMask stringset.Set + BruteWordList stringset.Set + BruteWordListMask stringset.Set + Blacklist stringset.Set + Domains stringset.Set + Excluded stringset.Set + Included stringset.Set MaxDNSQueries int MinForRecursive int - Names []string + Names stringset.Set Ports utils.ParseInts - Resolvers utils.ParseStrings + Resolvers stringset.Set Options struct { Active bool BruteForcing bool @@ -136,7 +137,18 @@ func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) { } func runEnumCommand(clArgs []string) { - var args enumArgs + args := enumArgs{ + AltWordList: stringset.New(), + AltWordListMask: stringset.New(), + BruteWordList: stringset.New(), + BruteWordListMask: stringset.New(), + Blacklist: stringset.New(), + Domains: stringset.New(), + Excluded: stringset.New(), + Included: stringset.New(), + Names: stringset.New(), + Resolvers: stringset.New(), + } var help1, help2 bool enumCommand := flag.NewFlagSet("enum", flag.ContinueOnError) @@ -171,10 +183,10 @@ func runEnumCommand(clArgs []string) { } if len(args.AltWordListMask) > 0 { - args.AltWordList = utils.UniqueAppend(args.AltWordList, args.AltWordListMask...) + args.AltWordList.Union(args.AltWordListMask) } if len(args.BruteWordListMask) > 0 { - args.BruteWordList = utils.UniqueAppend(args.BruteWordList, args.BruteWordListMask...) + args.BruteWordList.Union(args.BruteWordListMask) } // Some input validation if args.Options.Passive && (args.Options.IPs || args.Options.IPv4 || args.Options.IPv6) { @@ -202,12 +214,12 @@ func runEnumCommand(clArgs []string) { rLog, wLog := io.Pipe() e.Config.Log = log.New(wLog, "", log.Lmicroseconds) - + // Check if a configuration file was provided, and if so, load the settings if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, e.Config); err == nil { // Check if a config file was provided that has DNS resolvers specified if r, err := config.GetResolversFromSettings(f); err == nil && len(args.Resolvers) == 0 { - args.Resolvers = r + args.Resolvers = stringset.New(r...) } } else if args.Filepaths.ConfigFile != "" { r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err) @@ -221,7 +233,7 @@ func runEnumCommand(clArgs []string) { } if len(args.Resolvers) > 0 { - if err := e.Pool.SetResolvers(args.Resolvers); err != nil { + if err := e.Pool.SetResolvers(args.Resolvers.Slice()); err != nil { r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err) os.Exit(1) } @@ -435,7 +447,7 @@ func processEnumInputFiles(args *enumArgs) error { return fmt.Errorf("Failed to parse the brute force wordlist file: %v", err) } - args.BruteWordList = utils.UniqueAppend(args.BruteWordList, list...) + args.BruteWordList.InsertMany(list...) } } if !args.Options.NoAlts && len(args.Filepaths.AltWordlist) > 0 { @@ -445,7 +457,7 @@ func processEnumInputFiles(args *enumArgs) error { return fmt.Errorf("Failed to parse the alterations wordlist file: %v", err) } - args.AltWordList = utils.UniqueAppend(args.AltWordList, list...) + args.AltWordList.InsertMany(list...) } } if args.Filepaths.Blacklist != "" { @@ -453,21 +465,21 @@ func processEnumInputFiles(args *enumArgs) error { if err != nil { return fmt.Errorf("Failed to parse the blacklist file: %v", err) } - args.Blacklist = utils.UniqueAppend(args.Blacklist, list...) + args.Blacklist.InsertMany(list...) } if args.Filepaths.ExcludedSrcs != "" { list, err := config.GetListFromFile(args.Filepaths.ExcludedSrcs) if err != nil { return fmt.Errorf("Failed to parse the exclude file: %v", err) } - args.Excluded = utils.UniqueAppend(args.Excluded, list...) + args.Excluded.InsertMany(list...) } if args.Filepaths.IncludedSrcs != "" { list, err := config.GetListFromFile(args.Filepaths.IncludedSrcs) if err != nil { return fmt.Errorf("Failed to parse the include file: %v", err) } - args.Included = utils.UniqueAppend(args.Included, list...) + args.Included.InsertMany(list...) } if len(args.Filepaths.Names) > 0 { for _, f := range args.Filepaths.Names { @@ -476,7 +488,7 @@ func processEnumInputFiles(args *enumArgs) error { return fmt.Errorf("Failed to parse the subdomain names file: %v", err) } - args.Names = utils.UniqueAppend(args.Names, list...) + args.Names.InsertMany(list...) } } if len(args.Filepaths.Domains) > 0 { @@ -486,17 +498,17 @@ func processEnumInputFiles(args *enumArgs) error { return fmt.Errorf("Failed to parse the domain names file: %v", err) } - args.Domains = utils.UniqueAppend(args.Domains, list...) + args.Domains.InsertMany(list...) } } if len(args.Filepaths.Resolvers) > 0 { for _, f := range args.Filepaths.Resolvers { list, err := config.GetListFromFile(f) if err != nil { - return fmt.Errorf("Failed to parse the resolver file: %v", err) + return fmt.Errorf("Failed to parse the esolver file: %v", err) } - args.Resolvers = utils.UniqueAppend(args.Resolvers, list...) + args.Resolvers.InsertMany(list...) } } return nil @@ -523,13 +535,13 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error { e.Config.MaxDNSQueries = args.MaxDNSQueries } if len(args.BruteWordList) > 0 { - e.Config.Wordlist = args.BruteWordList + e.Config.Wordlist = args.BruteWordList.Slice() } if len(args.AltWordList) > 0 { - e.Config.AltWordlist = args.AltWordList + e.Config.AltWordlist = args.AltWordList.Slice() } if len(args.Names) > 0 { - e.ProvidedNames = args.Names + e.ProvidedNames = args.Names.Slice() } if args.Options.BruteForcing { e.Config.BruteForcing = true @@ -553,80 +565,43 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error { e.Config.Passive = true } if len(args.Blacklist) > 0 { - e.Config.Blacklist = args.Blacklist + e.Config.Blacklist = args.Blacklist.Slice() } disabled := compileDisabledSources(e.GetAllSourceNames(), args.Included, args.Excluded) if len(disabled) > 0 { - e.Config.DisabledDataSources = disabled + e.Config.DisabledDataSources = disabled.Slice() } // Attempt to add the provided domains to the configuration - e.Config.AddDomains(args.Domains) + e.Config.AddDomains(args.Domains.Slice()) if len(e.Config.Domains()) == 0 { return errors.New("No root domain names were provided") } return nil } -func compileDisabledSources(srcs []string, include, exclude []string) []string { - var inc, disable []string - - master := srcs - // Check that the include names are valid - if len(include) > 0 { - for _, incname := range include { - var found bool - - for _, name := range master { - if strings.EqualFold(name, incname) { - found = true - inc = append(inc, incname) - break - } - } +func compileDisabledSources(srcs []string, include, exclude stringset.Set) stringset.Set { + master := stringset.New(srcs...) - if !found { - r.Fprintf(color.Error, "%s is not an available data source\n", incname) - } - } - } // Check that the exclude names are valid - if len(exclude) > 0 { - for _, exclname := range exclude { - var found bool - - for _, name := range master { - if strings.EqualFold(name, exclname) { - found = true - disable = append(disable, exclname) - break - } - } - - if !found { - r.Fprintf(color.Error, "%s is not an available data source\n", exclname) - } - } + excLen := len(exclude) + exclude.Intersect(master) + if excLen != len(exclude) { + r.Fprintf(color.Error, "Invalid excluded data source specification\n") } - if len(inc) == 0 { - return disable + // Check that the include names are valid + incLen := len(include) + include.Intersect(master) + if incLen != len(include) { + r.Fprintf(color.Error, "Invalid included data source specification\n") } - // Data sources missing from the include list are disabled - for _, name := range master { - var found bool - for _, incname := range inc { - if strings.EqualFold(name, incname) { - found = true - break - } - } - - if !found { - disable = utils.UniqueAppend(disable, name) - } + if len(include) == 0 { + return exclude } - return disable + + master.Subtract(include) + return master } diff --git a/cmd/amass/intel.go b/cmd/amass/intel.go index efa69a765..218005ac2 100644 --- a/cmd/amass/intel.go +++ b/cmd/amass/intel.go @@ -19,6 +19,7 @@ import ( "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/intel" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/fatih/color" homedir "github.com/mitchellh/go-homedir" @@ -33,12 +34,12 @@ type intelArgs struct { ASNs utils.ParseInts CIDRs utils.ParseCIDRs OrganizationName string - Domains utils.ParseStrings - Excluded utils.ParseStrings - Included utils.ParseStrings + Domains stringset.Set + Excluded stringset.Set + Included stringset.Set MaxDNSQueries int Ports utils.ParseInts - Resolvers utils.ParseStrings + Resolvers stringset.Set Options struct { Active bool DemoMode bool @@ -97,7 +98,12 @@ func defineIntelFilepathFlags(intelFlags *flag.FlagSet, args *intelArgs) { } func runIntelCommand(clArgs []string) { - var args intelArgs + args := intelArgs{ + Domains: stringset.New(), + Excluded: stringset.New(), + Included: stringset.New(), + Resolvers: stringset.New(), + } var help1, help2 bool intelCommand := flag.NewFlagSet("intel", flag.ContinueOnError) @@ -159,7 +165,7 @@ func runIntelCommand(clArgs []string) { os.Exit(1) } - ic := intel.NewIntelCollection() + ic := intel.NewCollection() if ic == nil { r.Fprintf(color.Error, "%s\n", "No DNS resolvers passed the sanity check") os.Exit(1) @@ -167,12 +173,12 @@ func runIntelCommand(clArgs []string) { rLog, wLog := io.Pipe() ic.Config.Log = log.New(wLog, "", log.Lmicroseconds) - + // Check if a configuration file was provided, and if so, load the settings if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, ic.Config); err == nil { // Check if a config file was provided that has DNS resolvers specified if r, err := config.GetResolversFromSettings(f); err == nil && len(args.Resolvers) == 0 { - args.Resolvers = r + args.Resolvers = stringset.New(r...) } } else if args.Filepaths.ConfigFile != "" { r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err) @@ -186,7 +192,7 @@ func runIntelCommand(clArgs []string) { } if len(args.Resolvers) > 0 { - if err := ic.Pool.SetResolvers(args.Resolvers); err != nil { + if err := ic.Pool.SetResolvers(args.Resolvers.Slice()); err != nil { r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err) os.Exit(1) } @@ -210,7 +216,7 @@ func runIntelCommand(clArgs []string) { processIntelOutput(ic, &args, rLog) } -func processIntelOutput(ic *intel.IntelCollection, args *intelArgs, pipe *io.PipeReader) { +func processIntelOutput(ic *intel.Collection, args *intelArgs, pipe *io.PipeReader) { var err error // Prepare output file paths @@ -272,7 +278,7 @@ func processIntelOutput(ic *intel.IntelCollection, args *intelArgs, pipe *io.Pip } // If the user interrupts the program, print the summary information -func intelSignalHandler(ic *intel.IntelCollection) { +func intelSignalHandler(ic *intel.Collection) { quit := make(chan os.Signal, 1) signal.Notify(quit, os.Interrupt, syscall.SIGTERM) @@ -319,14 +325,14 @@ func processIntelInputFiles(args *intelArgs) error { if err != nil { return fmt.Errorf("Failed to parse the exclude file: %v", err) } - args.Excluded = utils.UniqueAppend(args.Excluded, list...) + args.Excluded.InsertMany(list...) } if args.Filepaths.IncludedSrcs != "" { list, err := config.GetListFromFile(args.Filepaths.IncludedSrcs) if err != nil { return fmt.Errorf("Failed to parse the include file: %v", err) } - args.Included = utils.UniqueAppend(args.Included, list...) + args.Included.InsertMany(list...) } if len(args.Filepaths.Domains) > 0 { for _, f := range args.Filepaths.Domains { @@ -335,7 +341,7 @@ func processIntelInputFiles(args *intelArgs) error { return fmt.Errorf("Failed to parse the domain names file: %v", err) } - args.Domains = utils.UniqueAppend(args.Domains, list...) + args.Domains.InsertMany(list...) } } if len(args.Filepaths.Resolvers) > 0 { @@ -345,14 +351,14 @@ func processIntelInputFiles(args *intelArgs) error { return fmt.Errorf("Failed to parse the resolver file: %v", err) } - args.Resolvers = utils.UniqueAppend(args.Resolvers, list...) + args.Resolvers.InsertMany(list...) } } return nil } // Setup the amass intelligence collection settings -func updateIntelConfiguration(ic *intel.IntelCollection, args *intelArgs) error { +func updateIntelConfiguration(ic *intel.Collection, args *intelArgs) error { if args.Options.Active { ic.Config.Active = true } @@ -377,9 +383,9 @@ func updateIntelConfiguration(ic *intel.IntelCollection, args *intelArgs) error disabled := compileDisabledSources(GetAllSourceNames(), args.Included, args.Excluded) if len(disabled) > 0 { - ic.Config.DisabledDataSources = disabled + ic.Config.DisabledDataSources = disabled.Slice() } // Attempt to add the provided domains to the configuration - ic.Config.AddDomains(args.Domains) + ic.Config.AddDomains(args.Domains.Slice()) return nil } diff --git a/cmd/amass/track.go b/cmd/amass/track.go index 317ce1ab4..81d3112cf 100644 --- a/cmd/amass/track.go +++ b/cmd/amass/track.go @@ -14,6 +14,7 @@ import ( "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/graph" "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/fatih/color" ) @@ -24,7 +25,7 @@ const ( ) type trackArgs struct { - Domains utils.ParseStrings + Domains stringset.Set Last int Since string Options struct { @@ -42,6 +43,8 @@ func runTrackCommand(clArgs []string) { var help1, help2 bool trackCommand := flag.NewFlagSet("track", flag.ContinueOnError) + args.Domains = stringset.New() + trackBuf := new(bytes.Buffer) trackCommand.SetOutput(trackBuf) @@ -84,7 +87,7 @@ func runTrackCommand(clArgs []string) { r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err) os.Exit(1) } - args.Domains = utils.UniqueAppend(args.Domains, list...) + args.Domains.InsertMany(list...) } if len(args.Domains) == 0 { r.Fprintln(color.Error, "No root domain names were provided") @@ -110,7 +113,7 @@ func runTrackCommand(clArgs []string) { args.Filepaths.Directory = cfg.Dir } if len(args.Domains) == 0 { - args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...) + args.Domains.InsertMany(cfg.Domains()...) } } else if args.Filepaths.ConfigFile != "" { r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err) @@ -126,7 +129,7 @@ func runTrackCommand(clArgs []string) { defer db.Close() // Obtain the enumerations that include the provided domain(s) - enums := enumIDs(args.Domains, db) + enums := enumIDs(args.Domains.Slice(), db) // There needs to be at least two enumerations to proceed if len(enums) < 2 { @@ -161,10 +164,10 @@ func runTrackCommand(clArgs []string) { latest = latest[:end] if args.Options.History { - completeHistoryOutput(args.Domains, enums, earliest, latest, db) + completeHistoryOutput(args.Domains.Slice(), enums, earliest, latest, db) return } - cumulativeOutput(args.Domains, enums, earliest, latest, db) + cumulativeOutput(args.Domains.Slice(), enums, earliest, latest, db) } func cumulativeOutput(domains []string, enums []string, ea, la []time.Time, db graph.DataHandler) { diff --git a/cmd/amass/viz.go b/cmd/amass/viz.go index ac2c28f63..0fc610c2c 100644 --- a/cmd/amass/viz.go +++ b/cmd/amass/viz.go @@ -16,7 +16,7 @@ import ( "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/graph" - "github.com/OWASP/Amass/utils" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils/viz" "github.com/fatih/color" ) @@ -26,7 +26,7 @@ const ( ) type vizArgs struct { - Domains utils.ParseStrings + Domains stringset.Set Enum int Options struct { D3 bool @@ -49,6 +49,8 @@ func runVizCommand(clArgs []string) { var help1, help2 bool vizCommand := flag.NewFlagSet("viz", flag.ContinueOnError) + args.Domains = stringset.New() + vizBuf := new(bytes.Buffer) vizCommand.SetOutput(vizBuf) @@ -94,7 +96,7 @@ func runVizCommand(clArgs []string) { r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err) return } - args.Domains = utils.UniqueAppend(args.Domains, list...) + args.Domains.InsertMany(list...) } if args.Filepaths.Output == "" { @@ -131,7 +133,7 @@ func runVizCommand(clArgs []string) { args.Filepaths.Directory = cfg.Dir } if len(args.Domains) == 0 { - args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...) + args.Domains.InsertMany(cfg.Domains()...) } } else if args.Filepaths.ConfigFile != "" { r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err) @@ -146,10 +148,10 @@ func runVizCommand(clArgs []string) { defer db.Close() if args.Enum > 0 { - uuid = enumIndexToID(args.Enum, args.Domains, db) + uuid = enumIndexToID(args.Enum, args.Domains.Slice(), db) } else { // Get the UUID for the most recent enumeration - uuid = mostRecentEnumID(args.Domains, db) + uuid = mostRecentEnumID(args.Domains.Slice(), db) } } if uuid == "" { diff --git a/config/config.go b/config/config.go index 8e23a895c..dfa0a7273 100644 --- a/config/config.go +++ b/config/config.go @@ -8,7 +8,6 @@ import ( "compress/gzip" "errors" "fmt" - homedir "github.com/mitchellh/go-homedir" "io" "log" "net" @@ -20,17 +19,20 @@ import ( "strings" "sync" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/go-ini/ini" "github.com/google/uuid" + homedir "github.com/mitchellh/go-homedir" ) const ( // DefaultOutputDirectory is the name of the directory used for output files, such as the graph database. DefaultOutputDirectory = "amass" - defaultWordlistURL = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/namelist.txt" - defaultAltWordlistURL = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/alterations.txt" + defaultConcurrentDNSQueries = 2500 + defaultWordlistURL = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/namelist.txt" + defaultAltWordlistURL = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/alterations.txt" ) // Config passes along Amass configuration settings and options. @@ -145,7 +147,7 @@ func (c *Config) CheckSettings() error { return errors.New("Active enumeration cannot be performed without DNS resolution") } if c.MaxDNSQueries <= 0 { - c.MaxDNSQueries = 1000 + c.MaxDNSQueries = defaultConcurrentDNSQueries } if len(c.Ports) == 0 { c.Ports = []int{443} @@ -212,16 +214,20 @@ func (c *Config) AddDomain(domain string) { return } } + // Check that the regular expression map has been initialized if c.regexps == nil { c.regexps = make(map[string]*regexp.Regexp) } + // Create the regular expression for this domain c.regexps[d] = utils.SubdomainRegex(d) if c.regexps[d] != nil { // Add the domain string to the list - c.domains = utils.UniqueAppend(c.domains, d) + c.domains = append(c.domains, d) } + + c.domains = stringset.Deduplicate(c.domains) } // Domains returns the list of domain names currently in the configuration. @@ -388,9 +394,11 @@ func (c *Config) loadBruteForceSettings(cfg *ini.File) error { if err != nil { return fmt.Errorf("Unable to load the file in the bruteforce wordlist_file setting: %s: %v", wordlist, err) } - c.Wordlist = utils.UniqueAppend(c.Wordlist, list...) + c.Wordlist = append(c.Wordlist, list...) } } + + c.Wordlist = stringset.Deduplicate(c.Wordlist) return nil } @@ -418,9 +426,11 @@ func (c *Config) loadAlterationSettings(cfg *ini.File) error { if err != nil { return fmt.Errorf("Unable to load the file in the alterations wordlist_file setting: %s: %v", wordlist, err) } - c.AltWordlist = utils.UniqueAppend(c.AltWordlist, list...) + c.AltWordlist = append(c.AltWordlist, list...) } } + + c.AltWordlist = stringset.Deduplicate(c.AltWordlist) return nil } @@ -455,13 +465,11 @@ func (c *Config) LoadSettings(path string) error { } // Load up all the blacklisted subdomain names if blacklisted, err := cfg.GetSection("blacklisted"); err == nil { - c.Blacklist = utils.UniqueAppend(c.Blacklist, - blacklisted.Key("subdomain").ValueWithShadows()...) + c.Blacklist = stringset.Deduplicate(blacklisted.Key("subdomain").ValueWithShadows()) } // Load up all the disabled data source names if disabled, err := cfg.GetSection("disabled_data_sources"); err == nil { - c.DisabledDataSources = utils.UniqueAppend( - c.DisabledDataSources, disabled.Key("data_source").ValueWithShadows()...) + c.DisabledDataSources = stringset.Deduplicate(disabled.Key("data_source").ValueWithShadows()) } // Load up all the Gremlin Server settings if gremlin, err := cfg.GetSection("gremlin"); err == nil { @@ -602,7 +610,9 @@ func GetListFromFile(path string) ([]string, error) { defer gzReader.Close() reader = gzReader } - return getWordList(reader) + + s, err := getWordList(reader) + return s, err } func getWordlistByURL(url string) ([]string, error) { @@ -624,7 +634,7 @@ func getWordList(reader io.Reader) ([]string, error) { words = append(words, w) } } - return words, nil + return stringset.Deduplicate(words), nil } func uniqueIntAppend(s []int, e string) []int { diff --git a/doc/install.md b/doc/install.md index aa6e9ac00..afd1db9fd 100644 --- a/doc/install.md +++ b/doc/install.md @@ -32,7 +32,15 @@ docker run -v ~/amass:/amass/ amass enum -brute -w /wordlists/all.txt -d example ## From Source -If you prefer to build your own binary from the latest release of the source code, make sure you have a correctly configured **Go >= 1.10** environment. More information about how to achieve this can be found [on the golang website.](https://golang.org/doc/install) Then, take the following steps: +If you prefer to build your own binary from the latest release of the source code, make sure you have a correctly configured **Go >= 1.12** environment. More information about how to achieve this can be found [on the golang website.](https://golang.org/doc/install). + +If you are not utilizing Go Modules, then you can simply execute the following command: + +```bash +go get -u github.com/OWASP/Amass/... +``` + +If you would like to build Amass using Go Modules to ensure the proper dependencies, then perform the following steps: 1. Download OWASP Amass: @@ -48,7 +56,7 @@ Ignore any error messages regarding what was pulled down. export GO111MODULE=on ``` -3. Next, build the binaries from the project source code: +3. Next, build the binary from the project source code: ```bash cd $GOPATH/src/github.com/OWASP/Amass @@ -56,9 +64,7 @@ cd $GOPATH/src/github.com/OWASP/Amass go install ./... ``` -At this point, the binaries should be in *$GOPATH/bin*. - -4. Several wordlists can be found in the following directory: +At this point, the binary should be in *$GOPATH/bin*. Several wordlists for performing DNS name alterations and brute forcing can be found in the following directory: ```bash ls $GOPATH/src/github.com/OWASP/Amass/wordlists/ @@ -66,9 +72,9 @@ ls $GOPATH/src/github.com/OWASP/Amass/wordlists/ ## Packages Maintained by the Amass Project -### Homebrew (macOS) +### Homebrew -For **Homebrew** on **Mac**, the following two commands will install Amass into your macOS environment: +For **Homebrew**, the following two commands will install Amass into your environment: ```bash brew tap caffix/amass diff --git a/enum/enum.go b/enum/enum.go index befadd4f3..0515aa2d2 100644 --- a/enum/enum.go +++ b/enum/enum.go @@ -84,7 +84,7 @@ func NewEnumeration() *Enumeration { if e.Pool == nil { return nil } - + e.dataSources = sources.GetAllSources(e.Config, e.Bus, e.Pool) return e } @@ -273,12 +273,15 @@ func (e *Enumeration) requiredServices() []services.Service { if e.Config.DataOptsWriter != nil { dms.AddDataHandler(graph.NewDataOptsHandler(e.Config.DataOptsWriter)) } - srvcs = append(srvcs, services.NewDNSService(e.Config, e.Bus, e.Pool), dms) + srvcs = append(srvcs, dms, services.NewDNSService(e.Config, e.Bus, e.Pool)) } namesrv := services.NewNameService(e.Config, e.Bus, e.Pool) namesrv.RegisterGraph(e.Graph) - srvcs = append(srvcs, namesrv, services.NewAddressService(e.Config, e.Bus, e.Pool)) + srvcs = append(srvcs, namesrv, + services.NewLogService(e.Config, e.Bus, e.Pool), + services.NewAddressService(e.Config, e.Bus, e.Pool), + ) if !e.Config.Passive { e.bruteSrv = services.NewBruteForceService(e.Config, e.Bus, e.Pool) diff --git a/examples/minimal.go b/examples/minimal.go index 46c3d74a0..eabef910c 100644 --- a/examples/minimal.go +++ b/examples/minimal.go @@ -14,15 +14,15 @@ func main() { e := enum.NewEnumeration() if e == nil { - return + return } - + go func() { for result := range e.Output { fmt.Println(result.Name) } }() - + // Setup the most basic amass configuration e.Config.AddDomain("example.com") e.Start() diff --git a/go.mod b/go.mod index 9b427c6c8..b8ce7783c 100644 --- a/go.mod +++ b/go.mod @@ -11,23 +11,25 @@ require ( github.com/caffix/cloudflare-roundtripper v0.0.0-20181218223503-4c29d231c9cb github.com/cayleygraph/cayley v0.7.5 github.com/cenkalti/backoff v2.2.1+incompatible // indirect - github.com/chromedp/cdproto v0.0.0-20190714225024-2508b04fd5cb // indirect + github.com/chromedp/cdproto v0.0.0-20190721111337-61a0348ea0b1 // indirect github.com/chromedp/chromedp v0.3.1 // indirect github.com/dghubble/go-twitter v0.0.0-20190719072343-39e5462e111f github.com/fatih/color v1.7.0 github.com/fpfeng/httpcache v0.0.0-20181220163524-ab6bbcc7c729 // indirect - github.com/geziyor/geziyor v0.0.0-20190714003752-df37629d4d5b + github.com/geziyor/geziyor v0.0.0-20190721090841-762854e5113a github.com/go-ini/ini v1.44.0 github.com/go-kit/kit v0.9.0 // indirect github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/gobwas/ws v1.0.2 // indirect github.com/gogo/protobuf v1.2.1 // indirect github.com/golang/snappy v0.0.1 // indirect + github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.0 // indirect github.com/irfansharif/cfilter v0.1.1 github.com/jmoiron/sqlx v1.2.0 github.com/johnnadratowski/golang-neo4j-bolt-driver v0.0.0-20181101021923-6b24c0085aae + github.com/json-iterator/go v1.1.7 // indirect github.com/kisielk/errcheck v1.2.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/lib/pq v1.2.0 @@ -50,11 +52,11 @@ require ( golang.org/x/exp v0.0.0-20190718202018-cfdd5522f6f6 // indirect golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9 // indirect golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 // indirect - golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect + golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect - golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386 // indirect - google.golang.org/grpc v1.22.0 // indirect + golang.org/x/sys v0.0.0-20190726002231-94b544f455ef // indirect + golang.org/x/tools v0.0.0-20190725161231-2e34cfcb95cb // indirect + google.golang.org/grpc v1.22.1 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/yaml.v2 v2.2.2 // indirect diff --git a/go.sum b/go.sum index afd918946..70d3ddc15 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,8 @@ github.com/chromedp/cdproto v0.0.0-20190609032908-dd39f0bf0a54 h1:2NlKweNkC3yy6I github.com/chromedp/cdproto v0.0.0-20190609032908-dd39f0bf0a54/go.mod h1:5NWqr1Ri5aJB5uSvUXfVpbBslleS+eMjspUWv2Lcaow= github.com/chromedp/cdproto v0.0.0-20190714225024-2508b04fd5cb h1:NzR98imHjC6LsmogFqMAOhJWqixFwnWcQQTc1qTJafU= github.com/chromedp/cdproto v0.0.0-20190714225024-2508b04fd5cb/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= +github.com/chromedp/cdproto v0.0.0-20190721111337-61a0348ea0b1 h1:TFdsyNN0dHau0EDzAnCv56ZgsAlucefia13zYRc18LA= +github.com/chromedp/cdproto v0.0.0-20190721111337-61a0348ea0b1/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0= github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05 h1:5iy45UjpWvkgTcd7GrGQSPr7sifrp9nNweI/eAsMjGE= github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05/go.mod h1:MsTqWB2yT7cErDFnF1F3y0PN8i/a/qQj+0GXKLW/I3s= github.com/chromedp/chromedp v0.3.1 h1:QV2NmprlDOMssqCjxemgHehVldLB2wp+Spa+bRbQGmc= @@ -58,6 +60,8 @@ github.com/geziyor/geziyor v0.0.0-20190703175417-71683ec6de71 h1:DD67izwjI7a9nMV github.com/geziyor/geziyor v0.0.0-20190703175417-71683ec6de71/go.mod h1:oL2vq/ZSixF5Wy8Ac+h0MazxgGgCCqvp2tXorUk+fqw= github.com/geziyor/geziyor v0.0.0-20190714003752-df37629d4d5b h1:fQWkKQc1pP88SLP2OoUWmPLGiWctkyU9RtzelL8hxbM= github.com/geziyor/geziyor v0.0.0-20190714003752-df37629d4d5b/go.mod h1:ByMdw1IoR4O5LTZW4LrkeYrqOb39uG9M6PFo1gjWKNE= +github.com/geziyor/geziyor v0.0.0-20190721090841-762854e5113a h1:MYb8yIB2bIkeNjHG0YtgvJrh+2IqEHCjymGowc+TYpg= +github.com/geziyor/geziyor v0.0.0-20190721090841-762854e5113a/go.mod h1:ByMdw1IoR4O5LTZW4LrkeYrqOb39uG9M6PFo1gjWKNE= github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38= github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w= @@ -101,9 +105,11 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -120,6 +126,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB github.com/johnnadratowski/golang-neo4j-bolt-driver v0.0.0-20181101021923-6b24c0085aae h1:YovtHFY2RXA6muqKixVlJlLBvBNNjiLTYeCYZTn/QDo= github.com/johnnadratowski/golang-neo4j-bolt-driver v0.0.0-20181101021923-6b24c0085aae/go.mod h1:xwUw3ZE1/D9drQgpluhRs4peTMKm1tQEZ4p7DrpyqwE= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -152,7 +159,9 @@ github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -240,6 +249,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= @@ -267,6 +278,8 @@ golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI= golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726002231-94b544f455ef h1:vwqipsjwy3Y8/PQk/LmiaFjos8aOnU6Tt6oRXKD3org= +golang.org/x/sys v0.0.0-20190726002231-94b544f455ef/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -287,6 +300,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190725161231-2e34cfcb95cb/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= @@ -303,6 +317,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/graph/graph.go b/graph/graph.go index f6436349e..804129761 100644 --- a/graph/graph.go +++ b/graph/graph.go @@ -16,7 +16,7 @@ import ( "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/requests" - "github.com/OWASP/Amass/utils" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils/viz" "github.com/cayleygraph/cayley" "github.com/cayleygraph/cayley/graph" @@ -445,7 +445,7 @@ func (g *Graph) EnumerationList() []string { it, _ := p.BuildIterator().Optimize() defer it.Close() - var ids []string + ids := stringset.New() ctx := context.TODO() for it.Next(ctx) { token := it.Result() @@ -453,10 +453,10 @@ func (g *Graph) EnumerationList() []string { label := quad.NativeOf(value).(string) if label != "" { - ids = utils.UniqueAppend(ids, label) + ids.Insert(label) } } - return ids + return ids.Slice() } // EnumerationDomains returns the domains that were involved in the provided enumeration. @@ -469,7 +469,7 @@ func (g *Graph) EnumerationDomains(uuid string) []string { it, _ := p.BuildIterator().Optimize() defer it.Close() - var domains []string + domains := stringset.New() ctx := context.TODO() for it.Next(ctx) { token := it.Result() @@ -477,10 +477,10 @@ func (g *Graph) EnumerationDomains(uuid string) []string { domain := quad.NativeOf(value).(string) if domain != "" { - domains = utils.UniqueAppend(domains, domain) + domains.Insert(domain) } } - return domains + return domains.Slice() } // EnumerationDateRange returns the date range associated with the provided enumeration UUID. @@ -588,7 +588,7 @@ func (g *Graph) getSubdomainNames(domain, uuid string, marked bool) []string { } func (g *Graph) getCNAMEs(sub, uuid string) []string { - names := []string{sub} + names := stringset.New(sub) cname := quad.String(sub) for i := 0; i < 10; i++ { @@ -598,9 +598,9 @@ func (g *Graph) getCNAMEs(sub, uuid string) []string { } // Traverse to the next CNAME cname = quad.String(target) - names = utils.UniqueAppend(names, target) + names.Insert(target) } - return names + return names.Slice() } func (g *Graph) buildOutput(sub, uuid string) *requests.Output { diff --git a/intel/intel.go b/intel/intel.go index ab3b12ece..2d5f75e04 100644 --- a/intel/intel.go +++ b/intel/intel.go @@ -23,8 +23,8 @@ import ( "github.com/OWASP/Amass/utils" ) -// IntelCollection is the object type used to execute a open source information gathering with Amass. -type IntelCollection struct { +// Collection is the object type used to execute a open source information gathering with Amass. +type Collection struct { Config *config.Config Bus *eb.EventBus Pool *resolvers.ResolverPool @@ -44,9 +44,9 @@ type IntelCollection struct { activeChan chan struct{} } -// NewIntelCollection returns an initialized IntelCollection object that has not been started yet. -func NewIntelCollection() *IntelCollection { - ic := &IntelCollection{ +// NewCollection returns an initialized Collection object that has not been started yet. +func NewCollection() *Collection { + c := &Collection{ Config: &config.Config{Log: log.New(ioutil.Discard, "", 0)}, Bus: eb.NewEventBus(), Pool: resolvers.NewResolverPool(nil), @@ -57,28 +57,28 @@ func NewIntelCollection() *IntelCollection { domainChan: make(chan *requests.Output, 100), activeChan: make(chan struct{}, 100), } - if ic.Pool == nil { + if c.Pool == nil { return nil } - return ic + return c } // HostedDomains uses open source intelligence to discover root domain names in the target infrastructure. -func (ic *IntelCollection) HostedDomains() error { - if ic.Output == nil { +func (c *Collection) HostedDomains() error { + if c.Output == nil { return errors.New("The intelligence collection did not have an output channel") - } else if err := ic.Config.CheckSettings(); err != nil { + } else if err := c.Config.CheckSettings(); err != nil { return err } - go ic.startAddressRanges() - go ic.processCIDRs() + go c.startAddressRanges() + go c.processCIDRs() go func() { - for _, cidr := range ic.Config.CIDRs { - ic.cidrChan <- cidr + for _, cidr := range c.Config.CIDRs { + c.cidrChan <- cidr } }() - ic.asnsToCIDRs() + c.asnsToCIDRs() var active bool filter := utils.NewStringFilter() @@ -86,50 +86,55 @@ func (ic *IntelCollection) HostedDomains() error { loop: for { select { - case <-ic.Done: + case <-c.Done: break loop case <-t.C: if !active { - close(ic.Done) + close(c.Done) } active = false - case <-ic.activeChan: + case <-c.activeChan: active = true - case d := <-ic.domainChan: + case d := <-c.domainChan: active = true if !filter.Duplicate(d.Domain) { - ic.Output <- d + c.Output <- d } } } t.Stop() - close(ic.Output) + close(c.Output) return nil } -func (ic *IntelCollection) startAddressRanges() { - for _, addr := range ic.Config.Addresses { - ic.Config.SemMaxDNSQueries.Acquire(1) - go ic.investigateAddr(addr.String()) +func (c *Collection) startAddressRanges() { + for _, addr := range c.Config.Addresses { + c.Config.SemMaxDNSQueries.Acquire(1) + go c.investigateAddr(addr.String()) } } -func (ic *IntelCollection) processCIDRs() { +func (c *Collection) processCIDRs() { for { select { - case <-ic.Done: + case <-c.Done: return - case cidr := <-ic.cidrChan: + case cidr := <-c.cidrChan: + // Skip IPv6 netblocks, since they are simply too large + if ip := cidr.IP.Mask(cidr.Mask); utils.IsIPv6(ip) { + continue + } + for _, addr := range utils.NetHosts(cidr) { - ic.Config.SemMaxDNSQueries.Acquire(1) - go ic.investigateAddr(addr.String()) + c.Config.SemMaxDNSQueries.Acquire(1) + go c.investigateAddr(addr.String()) } } } } -func (ic *IntelCollection) investigateAddr(addr string) { - defer ic.Config.SemMaxDNSQueries.Release(1) +func (c *Collection) investigateAddr(addr string) { + defer c.Config.SemMaxDNSQueries.Release(1) ip := net.ParseIP(addr) if ip == nil { @@ -137,10 +142,10 @@ func (ic *IntelCollection) investigateAddr(addr string) { } addrinfo := requests.AddressInfo{Address: ip} - ic.activeChan <- struct{}{} - if _, answer, err := ic.Pool.ReverseDNS(addr); err == nil { - if d := strings.TrimSpace(ic.Pool.SubdomainToDomain(answer)); d != "" { - ic.domainChan <- &requests.Output{ + c.activeChan <- struct{}{} + if _, answer, err := c.Pool.ReverseDNS(addr); err == nil { + if d := strings.TrimSpace(c.Pool.SubdomainToDomain(answer)); d != "" { + c.domainChan <- &requests.Output{ Name: d, Domain: d, Addresses: []requests.AddressInfo{addrinfo}, @@ -150,37 +155,37 @@ func (ic *IntelCollection) investigateAddr(addr string) { } } - ic.activeChan <- struct{}{} - if !ic.Config.Active { + c.activeChan <- struct{}{} + if !c.Config.Active { return } - for _, name := range utils.PullCertificateNames(addr, ic.Config.Ports) { + for _, name := range utils.PullCertificateNames(addr, c.Config.Ports) { if n := strings.TrimSpace(name); n != "" { - ic.domainChan <- &requests.Output{ + c.domainChan <- &requests.Output{ Name: n, - Domain: ic.Pool.SubdomainToDomain(n), + Domain: c.Pool.SubdomainToDomain(n), Addresses: []requests.AddressInfo{addrinfo}, Tag: requests.CERT, Source: "Active Cert", } } } - ic.activeChan <- struct{}{} + c.activeChan <- struct{}{} } -func (ic *IntelCollection) asnsToCIDRs() { - if len(ic.Config.ASNs) == 0 { +func (c *Collection) asnsToCIDRs() { + if len(c.Config.ASNs) == 0 { return } - ic.Bus.Subscribe(requests.NewASNTopic, ic.updateNetCache) - defer ic.Bus.Unsubscribe(requests.NewASNTopic, ic.updateNetCache) + c.Bus.Subscribe(requests.NewASNTopic, c.updateNetCache) + defer c.Bus.Unsubscribe(requests.NewASNTopic, c.updateNetCache) - srcs := sources.GetAllSources(ic.Config, ic.Bus, ic.Pool) + srcs := sources.GetAllSources(c.Config, c.Bus, c.Pool) // Select the data sources desired by the user - if len(ic.Config.DisabledDataSources) > 0 { - srcs = ExcludeDisabledDataSources(srcs, ic.Config) + if len(c.Config.DisabledDataSources) > 0 { + srcs = ExcludeDisabledDataSources(srcs, c.Config) } // Keep only the data sources that successfully start var keep []services.Service @@ -195,7 +200,7 @@ func (ic *IntelCollection) asnsToCIDRs() { srcs = keep // Send the ASN requests to the data sources - for _, asn := range ic.Config.ASNs { + for _, asn := range c.Config.ASNs { for _, src := range srcs { src.SendASNRequest(&requests.ASNRequest{ASN: asn}) } @@ -203,10 +208,10 @@ func (ic *IntelCollection) asnsToCIDRs() { t := time.NewTicker(5 * time.Second) defer t.Stop() - defer ic.sendNetblockCIDRs() + defer c.sendNetblockCIDRs() for { select { - case <-ic.Done: + case <-c.Done: return case <-t.C: done := true @@ -223,49 +228,49 @@ func (ic *IntelCollection) asnsToCIDRs() { } } -func (ic *IntelCollection) sendNetblockCIDRs() { - ic.netLock.Lock() - defer ic.netLock.Unlock() +func (c *Collection) sendNetblockCIDRs() { + c.netLock.Lock() + defer c.netLock.Unlock() filter := utils.NewStringFilter() - for _, record := range ic.netCache { - for _, netblock := range record.Netblocks { + for _, record := range c.netCache { + for netblock := range record.Netblocks { _, ipnet, err := net.ParseCIDR(netblock) if err == nil && !filter.Duplicate(ipnet.String()) { - ic.cidrChan <- ipnet + c.cidrChan <- ipnet } } } } -func (ic *IntelCollection) updateNetCache(req *requests.ASNRequest) { - ic.netLock.Lock() - defer ic.netLock.Unlock() +func (c *Collection) updateNetCache(req *requests.ASNRequest) { + c.netLock.Lock() + defer c.netLock.Unlock() - if _, found := ic.netCache[req.ASN]; !found { - ic.netCache[req.ASN] = req + if _, found := c.netCache[req.ASN]; !found { + c.netCache[req.ASN] = req return } - c := ic.netCache[req.ASN] + entry := c.netCache[req.ASN] // This is additional information for an ASN entry - if c.Prefix == "" && req.Prefix != "" { - c.Prefix = req.Prefix + if entry.Prefix == "" && req.Prefix != "" { + entry.Prefix = req.Prefix } - if c.CC == "" && req.CC != "" { - c.CC = req.CC + if entry.CC == "" && req.CC != "" { + entry.CC = req.CC } - if c.Registry == "" && req.Registry != "" { - c.Registry = req.Registry + if entry.Registry == "" && req.Registry != "" { + entry.Registry = req.Registry } - if c.AllocationDate.IsZero() && !req.AllocationDate.IsZero() { - c.AllocationDate = req.AllocationDate + if entry.AllocationDate.IsZero() && !req.AllocationDate.IsZero() { + entry.AllocationDate = req.AllocationDate } - if c.Description == "" && req.Description != "" { - c.Description = req.Description + if entry.Description == "" && req.Description != "" { + entry.Description = req.Description } - c.Netblocks = utils.UniqueAppend(c.Netblocks, req.Netblocks...) - ic.netCache[req.ASN] = c + entry.Netblocks.Union(req.Netblocks) + c.netCache[req.ASN] = entry } // LookupASNsByName returns requests.ASNRequest objects for autonomous systems with @@ -302,13 +307,13 @@ func LookupASNsByName(s string) ([]*requests.ASNRequest, error) { } // ReverseWhois returns domain names that are related to the domains provided -func (ic *IntelCollection) ReverseWhois() error { +func (c *Collection) ReverseWhois() error { filter := utils.NewStringFilter() collect := func(req *requests.WhoisRequest) { for _, d := range req.NewDomains { if !filter.Duplicate(d) { - ic.Output <- &requests.Output{ + c.Output <- &requests.Output{ Name: d, Domain: d, Tag: req.Tag, @@ -317,13 +322,13 @@ func (ic *IntelCollection) ReverseWhois() error { } } } - ic.Bus.Subscribe(requests.NewWhoisTopic, collect) - defer ic.Bus.Unsubscribe(requests.NewWhoisTopic, collect) + c.Bus.Subscribe(requests.NewWhoisTopic, collect) + defer c.Bus.Unsubscribe(requests.NewWhoisTopic, collect) - srcs := sources.GetAllSources(ic.Config, ic.Bus, ic.Pool) + srcs := sources.GetAllSources(c.Config, c.Bus, c.Pool) // Select the data sources desired by the user - if len(ic.Config.DisabledDataSources) > 0 { - srcs = ExcludeDisabledDataSources(srcs, ic.Config) + if len(c.Config.DisabledDataSources) > 0 { + srcs = ExcludeDisabledDataSources(srcs, c.Config) } // Keep only the data sources that successfully start var keep []services.Service @@ -338,7 +343,7 @@ func (ic *IntelCollection) ReverseWhois() error { srcs = keep // Send the whois requests to the data sources - for _, domain := range ic.Config.Domains() { + for _, domain := range c.Config.Domains() { for _, src := range srcs { src.SendWhoisRequest(&requests.WhoisRequest{Domain: domain}) } @@ -348,7 +353,7 @@ func (ic *IntelCollection) ReverseWhois() error { loop: for { select { - case <-ic.Done: + case <-c.Done: break loop case <-t.C: done := true @@ -364,7 +369,7 @@ loop: } } t.Stop() - close(ic.Output) + close(c.Output) return nil } diff --git a/requests/request.go b/requests/request.go index 8830cd39d..1611a8c17 100644 --- a/requests/request.go +++ b/requests/request.go @@ -6,6 +6,8 @@ package requests import ( "net" "time" + + "github.com/OWASP/Amass/stringset" ) // Request tag types. @@ -34,6 +36,7 @@ const ( NewASNTopic = "amass:asn" WhoisRequestTopic = "amass:whoisreq" NewWhoisTopic = "amass:whoisinfo" + LogTopic = "amass:log" ) // DNSAnswer is the type used by Amass to represent a DNS record. @@ -70,7 +73,7 @@ type ASNRequest struct { Registry string AllocationDate time.Time Description string - Netblocks []string + Netblocks stringset.Set Tag string Source string } diff --git a/resolvers/pool.go b/resolvers/pool.go index e25ae8eb2..4d20a62de 100644 --- a/resolvers/pool.go +++ b/resolvers/pool.go @@ -338,7 +338,7 @@ func (rp *ResolverPool) performElection(votes []*resolveVote, name, qtype string } if len(ans) == 0 { - return ans, &ResolveError{Err: fmt.Sprintf("DNS query for %s, type %d returned 0 records", name, qt)} + return ans, &ResolveError{Err: fmt.Sprintf("Resolver Pool: DNS query for %s type %d returned 0 records", name, qt)} } return ans, nil } diff --git a/resolvers/resolver.go b/resolvers/resolver.go index 71ebfb7f6..cb3c3fa18 100644 --- a/resolvers/resolver.go +++ b/resolvers/resolver.go @@ -62,6 +62,7 @@ func randomInt(min, max int) int { type Resolver struct { sync.RWMutex Address string + Port string WindowDuration time.Duration Dialer *net.Dialer Conn net.Conn @@ -80,19 +81,22 @@ type Resolver struct { // NewResolver initializes a Resolver that send DNS queries to the IP address in the addr value. func NewResolver(addr string) *Resolver { + port := "53" parts := strings.Split(addr, ":") - if len(parts) == 1 && parts[0] == addr { - addr += ":53" + if len(parts) == 2 { + addr = parts[0] + port = parts[1] } d := &net.Dialer{} - conn, err := d.Dial("udp", addr) + conn, err := d.Dial("udp", addr+":"+port) if err != nil { return nil } r := &Resolver{ Address: addr, + Port: port, WindowDuration: 2 * time.Second, Dialer: d, Conn: conn, @@ -339,7 +343,8 @@ func (r *Resolver) checkForTimeouts() { // Remove the timed out requests from the map for _, id := range timeouts { if req := r.pullRequest(id); req != nil { - estr := fmt.Sprintf("DNS query for %s, type %d timed out", req.Name, req.Qtype) + estr := fmt.Sprintf("DNS query on resolver %s, for %s type %d timed out", + r.Address, req.Name, req.Qtype) r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) } } @@ -423,10 +428,10 @@ func (r *Resolver) tcpExchange(req *resolveRequest) { msg := queryMessage(r.getID(), req.Name, req.Qtype) d := net.Dialer{Timeout: r.WindowDuration} - conn, err := d.Dial("tcp", r.Address) + conn, err := d.Dial("tcp", r.Address+":"+r.Port) if err != nil { r.pullRequest(msg.MsgHdr.Id) - estr := fmt.Sprintf("DNS: Failed to obtain TCP connection to %s: %v", r.Address, err) + estr := fmt.Sprintf("DNS: Failed to obtain TCP connection to %s: %v", r.Address+":"+r.Port, err) r.returnRequest(req, makeResolveResult(nil, true, estr, 100)) return } @@ -470,8 +475,8 @@ func (r *Resolver) processMessage(msg *dns.Msg) { break } } - estr := fmt.Sprintf("DNS query for %s, type %d returned error %s", - req.Name, req.Qtype, dns.RcodeToString[msg.Rcode]) + estr := fmt.Sprintf("DNS query on resolver %s, for %s type %d returned error %s", + r.Address, req.Name, req.Qtype, dns.RcodeToString[msg.Rcode]) r.returnRequest(req, makeResolveResult(nil, again, estr, msg.Rcode)) return } @@ -492,7 +497,8 @@ func (r *Resolver) processMessage(msg *dns.Msg) { } if len(answers) == 0 { - estr := fmt.Sprintf("DNS query for %s, type %d returned 0 records", req.Name, req.Qtype) + estr := fmt.Sprintf("DNS query on resolver %s, for %s type %d returned 0 records", + r.Address, req.Name, req.Qtype) r.returnRequest(req, makeResolveResult(nil, false, estr, msg.Rcode)) return } diff --git a/services/addrsrv.go b/services/addrsrv.go index a8429465f..a76c3bb1c 100644 --- a/services/addrsrv.go +++ b/services/addrsrv.go @@ -21,6 +21,7 @@ var ( // Cache for the infrastructure data collected from online sources netLock sync.Mutex netCache map[int]*requests.ASNRequest + // The reserved network address ranges reservedAddrRanges []*net.IPNet reservedCIDRs = []string{ @@ -45,8 +46,7 @@ var ( } ) -// AddressService is the Service that handles all newly discovered IP addresses -// within the architecture. This is achieved by receiving all the NEWADDR events. +// AddressService is the Service that handles all newly discovered IP addresses within the architecture. type AddressService struct { BaseService @@ -64,10 +64,10 @@ func init() { } // NewAddressService returns he object initialized, but not yet started. -func NewAddressService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AddressService { +func NewAddressService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AddressService { as := &AddressService{filter: utils.NewStringFilter()} - as.BaseService = *NewBaseService(as, "Address Service", c, bus, pool) + as.BaseService = *NewBaseService(as, "Address Service", cfg, bus, pool) return as } @@ -162,7 +162,7 @@ func (as *AddressService) updateConfigWithNetblocks(req *requests.ASNRequest) { filter.Duplicate(cidr.String()) } - for _, block := range req.Netblocks { + for block := range req.Netblocks { if filter.Duplicate(block) { continue } @@ -198,7 +198,7 @@ loop: break loop case a := <-asnchan: updateCache(a) - for _, block := range a.Netblocks { + for block := range a.Netblocks { if _, cidr, err := net.ParseCIDR(block); err == nil && cidr.Contains(ip) { break loop } @@ -242,7 +242,7 @@ func updateCache(req *requests.ASNRequest) { if c.Description == "" && req.Description != "" { c.Description = req.Description } - c.Netblocks = utils.UniqueAppend(c.Netblocks, req.Netblocks...) + c.Netblocks.Union(req.Netblocks) netCache[req.ASN] = c } @@ -260,7 +260,7 @@ func ipSearch(addr string) *requests.ASNRequest { var desc string ip := net.ParseIP(addr) for asn, record := range netCache { - for _, netblock := range record.Netblocks { + for netblock := range record.Netblocks { _, ipnet, err := net.ParseCIDR(netblock) if err != nil { continue diff --git a/services/brute.go b/services/brute.go index 0eb00a092..54be724f8 100644 --- a/services/brute.go +++ b/services/brute.go @@ -4,6 +4,7 @@ package services import ( + "fmt" "strings" "sync" "time" @@ -37,10 +38,10 @@ type BruteForceService struct { } // NewBruteForceService returns he object initialized, but not yet started. -func NewBruteForceService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BruteForceService { +func NewBruteForceService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BruteForceService { bfs := &BruteForceService{filter: utils.NewStringFilter()} - bfs.BaseService = *NewBaseService(bfs, "Brute Forcing", c, bus, pool) + bfs.BaseService = *NewBaseService(bfs, "Brute Forcing", cfg, bus, pool) return bfs } @@ -144,9 +145,10 @@ func (bfs *BruteForceService) performBruteForcing(subdomain, domain string) { bfs.Pool().GetWildcardType(req) == resolvers.WildcardTypeDynamic { return } + wordlist := bfs.Config().Wordlist bfs.totalLock.Lock() - bfs.totalNames += len(bfs.Config().Wordlist) + bfs.totalNames += len(wordlist) bfs.totalLock.Unlock() var idx int @@ -159,11 +161,11 @@ func (bfs *BruteForceService) performBruteForcing(subdomain, domain string) { case <-t.C: bfs.SetActive() default: - if idx >= len(bfs.Config().Wordlist) { + if idx >= len(wordlist) { return } bfs.Config().SemMaxDNSQueries.Acquire(1) - word := strings.ToLower(bfs.Config().Wordlist[idx]) + word := strings.ToLower(wordlist[idx]) go bfs.bruteForceResolution(word, subdomain, domain) idx++ } @@ -190,7 +192,7 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) { break } } else { - bfs.Config().Log.Printf("%s: %v", bfs.String(), err) + bfs.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", bfs.String(), err)) } bfs.metrics.QueryTime(time.Now()) bfs.SetActive() diff --git a/services/datamgmtsrv.go b/services/datamgmtsrv.go index 2a0b85fe0..c32b06089 100644 --- a/services/datamgmtsrv.go +++ b/services/datamgmtsrv.go @@ -4,6 +4,7 @@ package services import ( + "fmt" "regexp" "strings" "time" @@ -27,10 +28,10 @@ type DataManagerService struct { } // NewDataManagerService returns he object initialized, but not yet started. -func NewDataManagerService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DataManagerService { +func NewDataManagerService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DataManagerService { dms := &DataManagerService{domainFilter: utils.NewStringFilter()} - dms.BaseService = *NewBaseService(dms, "Data Manager", c, bus, pool) + dms.BaseService = *NewBaseService(dms, "Data Manager", cfg, bus, pool) return dms } @@ -116,7 +117,7 @@ func (dms *DataManagerService) insertDomain(domain string) { Source: "Forward DNS", }) if err != nil { - dms.Config().Log.Printf("%s failed to insert domain: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert domain: %v", handler, err)) } } dms.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ @@ -150,7 +151,7 @@ func (dms *DataManagerService) insertCNAME(req *requests.DNSRequest, recidx int) Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert CNAME: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert CNAME: %v", handler, err)) } } dms.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ @@ -178,7 +179,7 @@ func (dms *DataManagerService) insertA(req *requests.DNSRequest, recidx int) { Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert A record: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert A record: %v", handler, err)) } } dms.insertInfrastructure(addr) @@ -209,7 +210,7 @@ func (dms *DataManagerService) insertAAAA(req *requests.DNSRequest, recidx int) Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert AAAA record: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert AAAA record: %v", handler, err)) } } dms.insertInfrastructure(addr) @@ -245,7 +246,7 @@ func (dms *DataManagerService) insertPTR(req *requests.DNSRequest, recidx int) { Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert PTR record: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert PTR record: %v", handler, err)) } } dms.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ @@ -276,7 +277,7 @@ func (dms *DataManagerService) insertSRV(req *requests.DNSRequest, recidx int) { Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert SRV record: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert SRV record: %v", handler, err)) } } @@ -314,7 +315,7 @@ func (dms *DataManagerService) insertNS(req *requests.DNSRequest, recidx int) { Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert NS record: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert NS record: %v", handler, err)) } } if target != domain { @@ -350,7 +351,7 @@ func (dms *DataManagerService) insertMX(req *requests.DNSRequest, recidx int) { Source: req.Source, }) if err != nil { - dms.Config().Log.Printf("%s failed to insert MX record: %v", handler, err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert MX record: %v", handler, err)) } } if target != domain { @@ -409,7 +410,7 @@ func (dms *DataManagerService) findNamesAndAddresses(data, domain string) { func (dms *DataManagerService) insertInfrastructure(addr string) { asn, cidr, desc, err := IPRequest(addr, dms.Bus()) if err != nil { - dms.Config().Log.Printf("%s: %v", dms.String(), err) + dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", dms.String(), err)) return } @@ -424,7 +425,9 @@ func (dms *DataManagerService) insertInfrastructure(addr string) { Description: desc, }) if err != nil { - dms.Config().Log.Printf("%s: %s failed to insert infrastructure data: %v", dms.String(), handler, err) + dms.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s failed to insert infrastructure data: %v", dms.String(), handler, err), + ) } } } diff --git a/services/dnssrv.go b/services/dnssrv.go index 430f3955d..dcaedab2c 100644 --- a/services/dnssrv.go +++ b/services/dnssrv.go @@ -49,7 +49,7 @@ type DNSService struct { } // NewDNSService returns he object initialized, but not yet started. -func NewDNSService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DNSService { +func NewDNSService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DNSService { ds := &DNSService{filter: utils.NewStringFilter()} for _, n := range badSubnets { @@ -58,7 +58,7 @@ func NewDNSService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverP } } - ds.BaseService = *NewBaseService(ds, "DNS Service", c, bus, pool) + ds.BaseService = *NewBaseService(ds, "DNS Service", cfg, bus, pool) return ds } @@ -179,7 +179,7 @@ func (ds *DNSService) performDNSRequest(req *requests.DNSRequest) { break } } else { - ds.Config().Log.Printf("DNS: %v", err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: %v", err)) } ds.metrics.QueryTime(time.Now()) ds.SetActive() @@ -246,7 +246,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) { answers = append(answers, a) } } else { - ds.Config().Log.Printf("DNS: NS record query error: %s: %v", subdomain, err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: NS record query error: %s: %v", subdomain, err)) } ds.metrics.QueryTime(time.Now()) @@ -257,7 +257,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) { answers = append(answers, a) } } else { - ds.Config().Log.Printf("DNS: MX record query error: %s: %v", subdomain, err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: MX record query error: %s: %v", subdomain, err)) } ds.metrics.QueryTime(time.Now()) @@ -266,7 +266,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) { if ans, err := ds.Pool().Resolve(subdomain, "SOA", resolvers.PriorityHigh); err == nil { answers = append(answers, ans...) } else { - ds.Config().Log.Printf("DNS: SOA record query error: %s: %v", subdomain, err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: SOA record query error: %s: %v", subdomain, err)) } ds.metrics.QueryTime(time.Now()) @@ -275,7 +275,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) { if ans, err := ds.Pool().Resolve(subdomain, "SPF", resolvers.PriorityHigh); err == nil { answers = append(answers, ans...) } else { - ds.Config().Log.Printf("DNS: SPF record query error: %s: %v", subdomain, err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: SPF record query error: %s: %v", subdomain, err)) } ds.metrics.QueryTime(time.Now()) @@ -298,17 +298,17 @@ func (ds *DNSService) attemptZoneXFR(sub, domain, server string) { addr, err := ds.nameserverAddr(server) if addr == "" { - ds.Config().Log.Printf("DNS: Zone XFR failed: %v", err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone XFR failed: %v", err)) return } - requests, err := resolvers.ZoneTransfer(sub, domain, addr) + reqs, err := resolvers.ZoneTransfer(sub, domain, addr) if err != nil { - ds.Config().Log.Printf("DNS: Zone XFR failed: %s: %v", server, err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone XFR failed: %s: %v", server, err)) return } - for _, req := range requests { + for _, req := range reqs { ds.resolvedName(req) } } @@ -316,17 +316,17 @@ func (ds *DNSService) attemptZoneXFR(sub, domain, server string) { func (ds *DNSService) attemptZoneWalk(domain, server string) { addr, err := ds.nameserverAddr(server) if addr == "" { - ds.Config().Log.Printf("DNS: Zone Walk failed: %v", err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone Walk failed: %v", err)) return } - requests, err := resolvers.NsecTraversal(domain, addr) + reqs, err := resolvers.NsecTraversal(domain, addr) if err != nil { - ds.Config().Log.Printf("DNS: Zone Walk failed: %s: %v", server, err) + ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone Walk failed: %s: %v", server, err)) return } - for _, req := range requests { + for _, req := range reqs { ds.SendDNSRequest(req) } } diff --git a/services/logsrv.go b/services/logsrv.go new file mode 100644 index 000000000..3cffbac54 --- /dev/null +++ b/services/logsrv.go @@ -0,0 +1,71 @@ +// Copyright 2017 Jeff Foley. All rights reserved. +// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. + +package services + +import ( + "time" + + "github.com/OWASP/Amass/config" + eb "github.com/OWASP/Amass/eventbus" + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" + "github.com/OWASP/Amass/utils" +) + +// LogService is the Service that performs logging for the architecture. +type LogService struct { + BaseService + + queue *utils.Queue +} + +// NewLogService returns he object initialized, but not yet started. +func NewLogService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *LogService { + l := &LogService{queue: new(utils.Queue)} + + l.BaseService = *NewBaseService(l, "Log Service", cfg, bus, pool) + return l +} + +// OnStart implements the Service interface. +func (l *LogService) OnStart() error { + l.BaseService.OnStart() + + l.Bus().Subscribe(requests.LogTopic, l.queue.Append) + go l.processRequests() + return nil +} + +func (l *LogService) processRequests() { + t := time.NewTicker(2 * time.Second) + defer t.Stop() + + for { + select { + case <-l.PauseChan(): + <-l.ResumeChan() + case <-l.Quit(): + return + case <-t.C: + if !l.queue.Empty() { + l.writeLogs() + } + case <-l.DNSRequestChan(): + case <-l.AddrRequestChan(): + case <-l.ASNRequestChan(): + case <-l.WhoisRequestChan(): + } + } +} + +func (l *LogService) writeLogs() { + for { + msg, ok := l.queue.Next() + if !ok { + break + } + + l.Config().Log.Print(msg.(string)) + } +} diff --git a/services/markov.go b/services/markov.go index 1da064006..4e385eab3 100644 --- a/services/markov.go +++ b/services/markov.go @@ -55,7 +55,7 @@ type MarkovService struct { } // NewMarkovService returns he object initialized, but not yet started. -func NewMarkovService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *MarkovService { +func NewMarkovService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *MarkovService { m := &MarkovService{ subs: make(map[string]*requests.DNSRequest), inFilter: utils.NewStringFilter(), @@ -66,7 +66,7 @@ func NewMarkovService(c *config.Config, bus *eb.EventBus, pool *resolvers.Resolv }, } - m.BaseService = *NewBaseService(m, "Markov Model", c, bus, pool) + m.BaseService = *NewBaseService(m, "Markov Model", cfg, bus, pool) return m } diff --git a/services/namesrv.go b/services/namesrv.go index 716047d2d..e0372ce1a 100644 --- a/services/namesrv.go +++ b/services/namesrv.go @@ -16,6 +16,23 @@ import ( "github.com/OWASP/Amass/utils" ) +var ( + topNames = []string{ + "www", + "online", + "webserver", + "ns1", + "mail", + "smtp", + "webmail", + "prod", + "test", + "vpn", + "ftp", + "ssh", + } +) + type timesRequest struct { Subdomain string Times chan int @@ -119,6 +136,17 @@ func (ns *NameService) Resolved(req *requests.DNSRequest) { if ns.Config().IsDomainInScope(req.Name) { ns.checkSubdomain(req) + + if ns.Config().BruteForcing && ns.Config().Recursive { + for _, name := range topNames { + go ns.newNameEvent(&requests.DNSRequest{ + Name: name + "." + req.Name, + Domain: req.Domain, + Tag: requests.ALT, + Source: ns.String(), + }) + } + } } } diff --git a/services/sources/alienvault.go b/services/sources/alienvault.go index 4b41531b6..3cbbac9ce 100644 --- a/services/sources/alienvault.go +++ b/services/sources/alienvault.go @@ -16,6 +16,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -29,13 +30,13 @@ type AlienVault struct { } // NewAlienVault returns he object initialized, but not yet started. -func NewAlienVault(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AlienVault { +func NewAlienVault(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AlienVault { a := &AlienVault{ SourceType: requests.API, RateLimit: 100 * time.Millisecond, } - a.BaseService = *services.NewBaseService(a, "AlienVault", c, bus, pool) + a.BaseService = *services.NewBaseService(a, "AlienVault", cfg, bus, pool) return a } @@ -45,7 +46,7 @@ func (a *AlienVault) OnStart() error { a.API = a.Config().GetAPIKey(a.String()) if a.API == nil || a.API.Key == "" { - a.Config().Log.Printf("%s: API key data was not provided", a.String()) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", a.String())) } go a.processRequests() @@ -95,7 +96,7 @@ func (a *AlienVault) executeDNSQuery(domain string) { u := a.getURL(domain) + "passive_dns" page, err := utils.RequestWebPage(u, nil, a.getHeaders(), "", "") if err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) return } // Extract the subdomain names and IP addresses from the passive DNS information @@ -106,25 +107,25 @@ func (a *AlienVault) executeDNSQuery(domain string) { } `json:"passive_dns"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) return } else if len(m.Subdomains) == 0 { - a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), u) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u)) return } - var ips []string - var names []string + ips := stringset.New() + names := stringset.New() for _, sub := range m.Subdomains { n := strings.ToLower(sub.Hostname) if re.MatchString(n) { - names = append(names, n) - ips = append(ips, sub.IP) + names.Insert(n) + ips.Insert(sub.IP) } } - for _, name := range names { + for name := range names { a.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, @@ -133,7 +134,7 @@ func (a *AlienVault) executeDNSQuery(domain string) { }) } - for _, ip := range ips { + for ip := range ips { a.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{ Address: ip, Tag: a.SourceType, @@ -153,7 +154,7 @@ func (a *AlienVault) executeURLQuery(domain string) { u := a.getURL(domain) + "url_list" page, err := utils.RequestWebPage(u, nil, headers, "", "") if err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) return } // Extract the subdomain names and IP addresses from the URL information @@ -173,22 +174,22 @@ func (a *AlienVault) executeURLQuery(domain string) { } `json:"url_list"` } if err := json.Unmarshal([]byte(page), &urls); err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) return } else if len(urls.URLs) == 0 { - a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), u) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u)) return } - var ips []string - var names []string + ips := stringset.New() + names := stringset.New() for _, u := range urls.URLs { n := strings.ToLower(u.Hostname) if re.MatchString(n) { - names = utils.UniqueAppend(names, n) + names.Insert(n) if u.Result.Worker.IP != "" { - ips = utils.UniqueAppend(ips, u.Result.Worker.IP) + ips.Insert(u.Result.Worker.IP) } } } @@ -200,15 +201,17 @@ func (a *AlienVault) executeURLQuery(domain string) { pageURL := u + "?page=" + strconv.Itoa(cur) page, err = utils.RequestWebPage(pageURL, nil, headers, "", "") if err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err)) break } if err := json.Unmarshal([]byte(page), &urls); err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err)) break } else if len(urls.URLs) == 0 { - a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), pageURL) + a.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The query returned zero results", a.String(), pageURL), + ) break } @@ -216,16 +219,16 @@ func (a *AlienVault) executeURLQuery(domain string) { n := strings.ToLower(u.Hostname) if re.MatchString(n) { - names = utils.UniqueAppend(names, n) + names.Insert(n) if u.Result.Worker.IP != "" { - ips = utils.UniqueAppend(ips, u.Result.Worker.IP) + ips.Insert(u.Result.Worker.IP) } } } } } - for _, name := range names { + for name := range names { a.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, @@ -234,7 +237,7 @@ func (a *AlienVault) executeURLQuery(domain string) { }) } - for _, ip := range ips { + for ip := range ips { a.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{ Address: ip, Tag: a.SourceType, @@ -244,14 +247,14 @@ func (a *AlienVault) executeURLQuery(domain string) { } func (a *AlienVault) queryWhoisForEmails(domain string) []string { - var emails []string + emails := stringset.New() u := a.getWhoisURL(domain) a.SetActive() page, err := utils.RequestWebPage(u, nil, a.getHeaders(), "", "") if err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) - return emails + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) + return emails.Slice() } var m struct { @@ -263,11 +266,13 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string { } `json:"data"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) - return emails + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) + return emails.Slice() } else if m.Count == 0 { - a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), u) - return emails + a.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u), + ) + return emails.Slice() } for _, row := range m.Data { @@ -282,25 +287,25 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string { // Unfortunately AlienVault doesn't categorize the email addresses so we // have to filter by something we know to avoid adding registrar emails if a.Config().IsDomainInScope(d) { - emails = utils.UniqueAppend(emails, email) + emails.Insert(email) } } } - return emails + return emails.Slice() } func (a *AlienVault) executeWhoisQuery(domain string) { emails := a.queryWhoisForEmails(domain) time.Sleep(a.RateLimit) - var newDomains []string + newDomains := stringset.New() headers := a.getHeaders() for _, email := range emails { a.SetActive() pageURL := a.getReverseWhoisURL(email) page, err := utils.RequestWebPage(pageURL, nil, headers, "", "") if err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err)) continue } @@ -309,25 +314,27 @@ func (a *AlienVault) executeWhoisQuery(domain string) { } var domains []record if err := json.Unmarshal([]byte(page), &domains); err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err)) continue } for _, d := range domains { if !a.Config().IsDomainInScope(d.Domain) { - newDomains = utils.UniqueAppend(newDomains, d.Domain) + newDomains.Insert(d.Domain) } } time.Sleep(a.RateLimit) } if len(newDomains) == 0 { - a.Config().Log.Printf("%s: Reverse whois failed to discover new domain names for %s", a.String(), domain) + a.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Reverse whois failed to discover new domain names for %s", a.String(), domain), + ) return } a.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{ Domain: domain, - NewDomains: newDomains, + NewDomains: newDomains.Slice(), Tag: a.SourceType, Source: a.String(), }) diff --git a/services/sources/alienvault_test.go b/services/sources/alienvault_test.go new file mode 100644 index 000000000..8c96e35c1 --- /dev/null +++ b/services/sources/alienvault_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" + + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" +) + +func TestAlienVault(t *testing.T) { + if *networkTest == false || *configPath == "" { + return + } + + cfg := setupConfig(domainTest) + api := cfg.GetAPIKey("alienvault") + if api == nil || api.Key == "" { + t.Errorf("API key data was not provided") + return + } + + bus, out := setupEventBus(requests.NewNameTopic) + defer bus.Stop() + + pool := resolvers.NewResolverPool(nil) + defer pool.Stop() + + srv := NewAlienVault(cfg, bus, pool) + + result := testService(srv, out) + if result < expectedTest { + t.Errorf("Found %d names, expected at least %d instead", result, expectedTest) + } +} diff --git a/services/sources/archiveit.go b/services/sources/archiveit.go index 0e4824bd6..edb17cf68 100644 --- a/services/sources/archiveit.go +++ b/services/sources/archiveit.go @@ -4,6 +4,8 @@ package sources import ( + "fmt" + "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" @@ -23,7 +25,7 @@ type ArchiveIt struct { } // NewArchiveIt returns he object initialized, but not yet started. -func NewArchiveIt(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveIt { +func NewArchiveIt(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveIt { a := &ArchiveIt{ domain: "wayback.archive-it.org", baseURL: "https://wayback.archive-it.org/all", @@ -31,7 +33,7 @@ func NewArchiveIt(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPo filter: utils.NewStringFilter(), } - a.BaseService = *services.NewBaseService(a, "ArchiveIt", c, bus, pool) + a.BaseService = *services.NewBaseService(a, "ArchiveIt", cfg, bus, pool) return a } @@ -67,7 +69,7 @@ func (a *ArchiveIt) executeQuery(sn, domain string) { names, err := crawl(a, a.baseURL, a.domain, sn, domain) if err != nil { - a.Config().Log.Printf("%s: %v", a.String(), err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", a.String(), err)) return } diff --git a/services/sources/archivetoday.go b/services/sources/archivetoday.go index ade2f8634..8d53e2b22 100644 --- a/services/sources/archivetoday.go +++ b/services/sources/archivetoday.go @@ -4,6 +4,8 @@ package sources import ( + "fmt" + "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" @@ -23,7 +25,7 @@ type ArchiveToday struct { } // NewArchiveToday returns he object initialized, but not yet started. -func NewArchiveToday(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveToday { +func NewArchiveToday(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveToday { a := &ArchiveToday{ domain: "archive.is", baseURL: "http://archive.is", @@ -31,7 +33,7 @@ func NewArchiveToday(c *config.Config, bus *eb.EventBus, pool *resolvers.Resolve filter: utils.NewStringFilter(), } - a.BaseService = *services.NewBaseService(a, "ArchiveToday", c, bus, pool) + a.BaseService = *services.NewBaseService(a, "ArchiveToday", cfg, bus, pool) return a } @@ -67,7 +69,7 @@ func (a *ArchiveToday) executeQuery(sn, domain string) { names, err := crawl(a, a.baseURL, a.domain, sn, domain) if err != nil { - a.Config().Log.Printf("%s: %v", a.String(), err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", a.String(), err)) return } diff --git a/services/sources/arquivo.go b/services/sources/arquivo.go index 8963b40ce..b9399985d 100644 --- a/services/sources/arquivo.go +++ b/services/sources/arquivo.go @@ -4,6 +4,8 @@ package sources import ( + "fmt" + "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" @@ -23,7 +25,7 @@ type Arquivo struct { } // NewArquivo returns he object initialized, but not yet started. -func NewArquivo(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Arquivo { +func NewArquivo(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Arquivo { a := &Arquivo{ domain: "arquivo.pt", baseURL: "http://arquivo.pt/wayback", @@ -31,7 +33,7 @@ func NewArquivo(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool filter: utils.NewStringFilter(), } - a.BaseService = *services.NewBaseService(a, "Arquivo", c, bus, pool) + a.BaseService = *services.NewBaseService(a, "Arquivo", cfg, bus, pool) return a } @@ -67,7 +69,7 @@ func (a *Arquivo) executeQuery(sn, domain string) { names, err := crawl(a, a.baseURL, a.domain, sn, domain) if err != nil { - a.Config().Log.Printf("%s: %v", a.String(), err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", a.String(), err)) return } diff --git a/services/sources/ask.go b/services/sources/ask.go index 9ba24bd7f..b848b8a1c 100644 --- a/services/sources/ask.go +++ b/services/sources/ask.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "strconv" "time" @@ -26,14 +27,14 @@ type Ask struct { } // NewAsk returns he object initialized, but not yet started. -func NewAsk(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Ask { +func NewAsk(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Ask { a := &Ask{ quantity: 10, // ask.com appears to be hardcoded at 10 results per page limit: 100, SourceType: requests.SCRAPE, } - a.BaseService = *services.NewBaseService(a, "Ask", c, bus, pool) + a.BaseService = *services.NewBaseService(a, "Ask", cfg, bus, pool) return a } @@ -81,7 +82,7 @@ func (a *Ask) executeQuery(domain string) { u := a.urlByPageNum(domain, i) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - a.Config().Log.Printf("%s: %s: %v", a.String(), u, err) + a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err)) return } diff --git a/services/sources/baidu.go b/services/sources/baidu.go index d24613521..74f0be501 100644 --- a/services/sources/baidu.go +++ b/services/sources/baidu.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "strconv" "time" @@ -20,20 +21,20 @@ import ( type Baidu struct { services.BaseService + SourceType string quantity int limit int - SourceType string } // NewBaidu returns he object initialized, but not yet started. -func NewBaidu(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Baidu { +func NewBaidu(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Baidu { b := &Baidu{ + SourceType: requests.SCRAPE, quantity: 20, limit: 100, - SourceType: requests.SCRAPE, } - b.BaseService = *services.NewBaseService(b, "Baidu", c, bus, pool) + b.BaseService = *services.NewBaseService(b, "Baidu", cfg, bus, pool) return b } @@ -81,7 +82,7 @@ func (b *Baidu) executeQuery(domain string) { u := b.urlByPageNum(domain, i) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - b.Config().Log.Printf("%s: %s: %v", b.String(), u, err) + b.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", b.String(), u, err)) return } diff --git a/services/sources/binaryedge.go b/services/sources/binaryedge.go index eeb5c9609..95c6ce505 100644 --- a/services/sources/binaryedge.go +++ b/services/sources/binaryedge.go @@ -5,6 +5,7 @@ package sources import ( "encoding/json" + "fmt" "time" "github.com/OWASP/Amass/config" @@ -25,13 +26,13 @@ type BinaryEdge struct { } // NewBinaryEdge returns he object initialized, but not yet started. -func NewBinaryEdge(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BinaryEdge { +func NewBinaryEdge(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BinaryEdge { be := &BinaryEdge{ SourceType: requests.API, RateLimit: 2 * time.Second, } - be.BaseService = *services.NewBaseService(be, "BinaryEdge", c, bus, pool) + be.BaseService = *services.NewBaseService(be, "BinaryEdge", cfg, bus, pool) return be } @@ -41,7 +42,7 @@ func (be *BinaryEdge) OnStart() error { be.API = be.Config().GetAPIKey(be.String()) if be.API == nil || be.API.Key == "" { - be.Config().Log.Printf("%s: API key data was not provided", be.String()) + be.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", be.String())) } go be.processRequests() @@ -90,7 +91,7 @@ func (be *BinaryEdge) executeQuery(domain string) { be.SetActive() page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - be.Config().Log.Printf("%s: %s: %v", be.String(), url, err) + be.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", be.String(), url, err)) return } // Extract the subdomain names from the REST API results diff --git a/services/sources/binaryedge_test.go b/services/sources/binaryedge_test.go new file mode 100644 index 000000000..a5fc81352 --- /dev/null +++ b/services/sources/binaryedge_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" + + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" +) + +func TestBinaryEdge(t *testing.T) { + if *networkTest == false || *configPath == "" { + return + } + + cfg := setupConfig(domainTest) + api := cfg.GetAPIKey("binaryedge") + if api == nil || api.Key == "" { + t.Errorf("API key data was not provided") + return + } + + bus, out := setupEventBus(requests.NewNameTopic) + defer bus.Stop() + + pool := resolvers.NewResolverPool(nil) + defer pool.Stop() + + srv := NewBinaryEdge(cfg, bus, pool) + + result := testService(srv, out) + if result < expectedTest { + t.Errorf("Found %d names, expected at least %d instead", result, expectedTest) + } +} diff --git a/services/sources/bing.go b/services/sources/bing.go index 5ea0f422a..9834834bd 100644 --- a/services/sources/bing.go +++ b/services/sources/bing.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "strconv" "time" @@ -26,14 +27,14 @@ type Bing struct { } // NewBing returns he object initialized, but not yet started. -func NewBing(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Bing { +func NewBing(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Bing { b := &Bing{ quantity: 20, limit: 200, SourceType: requests.SCRAPE, } - b.BaseService = *services.NewBaseService(b, "Bing", c, bus, pool) + b.BaseService = *services.NewBaseService(b, "Bing", cfg, bus, pool) return b } @@ -81,7 +82,7 @@ func (b *Bing) executeQuery(domain string) { u := b.urlByPageNum(domain, i) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - b.Config().Log.Printf("%s: %s: %v", b.String(), u, err) + b.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", b.String(), u, err)) return } diff --git a/services/sources/bufferover.go b/services/sources/bufferover.go index b550b087c..f2a941beb 100644 --- a/services/sources/bufferover.go +++ b/services/sources/bufferover.go @@ -22,10 +22,10 @@ type BufferOver struct { } // NewBufferOver returns he object initialized, but not yet started. -func NewBufferOver(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BufferOver { +func NewBufferOver(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BufferOver { b := &BufferOver{SourceType: requests.API} - b.BaseService = *services.NewBaseService(b, "BufferOver", c, bus, pool) + b.BaseService = *services.NewBaseService(b, "BufferOver", cfg, bus, pool) return b } @@ -63,7 +63,7 @@ func (b *BufferOver) executeQuery(domain string) { url := b.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - b.Config().Log.Printf("%s: %s: %v", b.String(), url, err) + b.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", b.String(), url, err)) return } diff --git a/services/sources/censys.go b/services/sources/censys.go index 61e4313af..1d95173e6 100644 --- a/services/sources/censys.go +++ b/services/sources/censys.go @@ -44,7 +44,7 @@ func (c *Censys) OnStart() error { c.API = c.Config().GetAPIKey(c.String()) if c.API == nil || c.API.Key == "" || c.API.Secret == "" { - c.Config().Log.Printf("%s: API key data was not provided", c.String()) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", c.String())) } go c.processRequests() @@ -101,7 +101,7 @@ func (c *Censys) apiQuery(domain string) { headers := map[string]string{"Content-Type": "application/json"} resp, err := utils.RequestWebPage(u, body, headers, c.API.Key, c.API.Secret) if err != nil { - c.Config().Log.Printf("%s: %s: %v", c.String(), u, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), u, err)) break } // Extract the subdomain names from the certificate information @@ -116,10 +116,12 @@ func (c *Censys) apiQuery(domain string) { } `json:"results"` } if err := json.Unmarshal([]byte(resp), &m); err != nil || m.Status != "ok" { - c.Config().Log.Printf("%s: %s: %v", c.String(), u, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), u, err)) break } else if len(m.Results) == 0 { - c.Config().Log.Printf("%s: %s: The query returned zero results", c.String(), u) + c.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The query returned zero results", c.String(), u), + ) break } @@ -163,7 +165,7 @@ func (c *Censys) executeQuery(domain string) { url = c.webURL(domain) page, err = utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - c.Config().Log.Printf("%s: %s: %v", c.String(), url, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err)) return } diff --git a/services/sources/certspotter.go b/services/sources/certspotter.go index 326e6bb83..f11e0646b 100644 --- a/services/sources/certspotter.go +++ b/services/sources/certspotter.go @@ -5,6 +5,7 @@ package sources import ( "encoding/json" + "fmt" "net/url" "time" @@ -76,7 +77,7 @@ func (c *CertSpotter) executeQuery(domain string) { url := c.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - c.Config().Log.Printf("%s: %s: %v", c.String(), url, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err)) return } // Extract the subdomain names from the certificate information diff --git a/services/sources/circl.go b/services/sources/circl.go index da4f458b1..ee6d7e878 100644 --- a/services/sources/circl.go +++ b/services/sources/circl.go @@ -6,6 +6,7 @@ package sources import ( "bufio" "encoding/json" + "fmt" "strings" "time" @@ -14,6 +15,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -43,7 +45,7 @@ func (c *CIRCL) OnStart() error { c.API = c.Config().GetAPIKey(c.String()) if c.API == nil || c.API.Username == "" || c.API.Password == "" { - c.Config().Log.Printf("%s: API key data was not provided", c.String()) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", c.String())) } go c.processRequests() @@ -83,7 +85,7 @@ func (c *CIRCL) executeQuery(domain string) { headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, c.API.Username, c.API.Password) if err != nil { - c.Config().Log.Printf("%s: %s: %v", c.String(), url, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err)) return } @@ -95,7 +97,7 @@ func (c *CIRCL) restURL(domain string) string { } func (c *CIRCL) passiveDNSJSON(page, domain string) { - var unique []string + unique := stringset.New() re := c.Config().DomainRegex(domain) if re == nil { @@ -119,11 +121,11 @@ func (c *CIRCL) passiveDNSJSON(page, domain string) { continue } if re.MatchString(j.Name) { - unique = utils.UniqueAppend(unique, j.Name) + unique.Insert(j.Name) } } - for _, name := range unique { + for name := range unique { c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, diff --git a/services/sources/commoncrawl.go b/services/sources/commoncrawl.go index cc341b0aa..358677b1d 100644 --- a/services/sources/commoncrawl.go +++ b/services/sources/commoncrawl.go @@ -4,7 +4,11 @@ package sources import ( + "bufio" + "encoding/json" + "fmt" "net/url" + "strings" "time" "github.com/OWASP/Amass/config" @@ -15,37 +19,19 @@ import ( "github.com/OWASP/Amass/utils" ) -var ( - commonCrawlIndexes = []string{ - "CC-MAIN-2019-04", - "CC-MAIN-2018-47", - "CC-MAIN-2018-39", - "CC-MAIN-2018-17", - "CC-MAIN-2018-05", - "CC-MAIN-2017-43", - "CC-MAIN-2017-26", - "CC-MAIN-2017-17", - "CC-MAIN-2017-04", - "CC-MAIN-2016-44", - "CC-MAIN-2016-26", - "CC-MAIN-2016-18", - } -) +const commonCrawlIndexListURL = "https://index.commoncrawl.org/collinfo.json" // CommonCrawl is the Service that handles access to the CommonCrawl data source. type CommonCrawl struct { services.BaseService - baseURL string SourceType string + indexURLs []string } // NewCommonCrawl returns he object initialized, but not yet started. func NewCommonCrawl(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *CommonCrawl { - c := &CommonCrawl{ - baseURL: "http://index.commoncrawl.org/", - SourceType: requests.SCRAPE, - } + c := &CommonCrawl{SourceType: requests.API} c.BaseService = *services.NewBaseService(c, "CommonCrawl", cfg, bus, pool) return c @@ -55,6 +41,33 @@ func NewCommonCrawl(cfg *config.Config, bus *eb.EventBus, pool *resolvers.Resolv func (c *CommonCrawl) OnStart() error { c.BaseService.OnStart() + // Get all of the index API URLs + page, err := utils.RequestWebPage(commonCrawlIndexListURL, nil, nil, "", "") + if err != nil { + c.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to obtain the index list: %v", c.String(), err), + ) + return fmt.Errorf("%s: Failed to obtain the index list: %v", c.String(), err) + } + + type index struct { + ID string `json:"id"` + Name string `json:"name"` + URL string `json:"cdx-api"` + } + + var indexList []index + if err := json.Unmarshal([]byte(page), &indexList); err != nil { + c.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to unmarshal the index list: %v", c.String(), err), + ) + return fmt.Errorf("%s: Failed to unmarshal the index list: %v", c.String(), err) + } + + for _, i := range indexList { + c.indexURLs = append(c.indexURLs, i.URL) + } + go c.processRequests() return nil } @@ -76,46 +89,79 @@ func (c *CommonCrawl) processRequests() { } func (c *CommonCrawl) executeQuery(domain string) { + filter := utils.NewStringFilter() re := c.Config().DomainRegex(domain) if re == nil { return } - t := time.NewTicker(time.Second) + t := time.NewTicker(500 * time.Millisecond) defer t.Stop() - for _, index := range commonCrawlIndexes { + for _, index := range c.indexURLs { c.SetActive() select { case <-c.Quit(): return case <-t.C: - u := c.getURL(index, domain) + u := c.getURL(domain, index) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - c.Config().Log.Printf("%s: %s: %v", c.String(), u, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), u, err)) continue } - for _, sd := range re.FindAllString(page, -1) { - c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ - Name: cleanName(sd), - Domain: domain, - Tag: c.SourceType, - Source: c.String(), - }) + for _, url := range c.parseJSON(page) { + if name := re.FindString(url); name != "" && !filter.Duplicate(name) { + c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ + Name: name, + Domain: domain, + Tag: c.SourceType, + Source: c.String(), + }) + } } } } } -func (c *CommonCrawl) getURL(index, domain string) string { - u, _ := url.Parse(c.baseURL + index + "-index") +func (c *CommonCrawl) parseJSON(page string) []string { + var urls []string + filter := utils.NewStringFilter() + + scanner := bufio.NewScanner(strings.NewReader(page)) + for scanner.Scan() { + // Get the next line of JSON + line := scanner.Text() + if line == "" { + continue + } + + var m struct { + URL string `json:"url"` + } + err := json.Unmarshal([]byte(line), &m) + if err != nil { + continue + } + + if !filter.Duplicate(m.URL) { + urls = append(urls, m.URL) + } + } + return urls +} + +func (c *CommonCrawl) getURL(domain, index string) string { + u, _ := url.Parse(index) u.RawQuery = url.Values{ - "url": {"*." + domain}, - "output": {"json"}, + "url": {"*." + domain}, + "output": {"json"}, + "filter": {"=status:200"}, + "fl": {"url,status"}, + "pageSize": {"2000"}, }.Encode() return u.String() } diff --git a/services/sources/crtsh.go b/services/sources/crtsh.go index 3e93bc3e7..1eeda84fe 100644 --- a/services/sources/crtsh.go +++ b/services/sources/crtsh.go @@ -5,13 +5,14 @@ package sources import ( "encoding/json" - "strings" + "fmt" "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" // Need the postgres driver @@ -44,7 +45,9 @@ func (c *Crtsh) OnStart() error { var err error c.db, err = sqlx.Connect("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable") if err != nil { - c.Config().Log.Printf("%s: Failed to connect to the database server: %v", c.String(), err) + c.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to connect to the database server: %v", c.String(), err), + ) c.haveConnection = false } @@ -86,18 +89,18 @@ func (c *Crtsh) executeQuery(domain string) { WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) ORDER BY ci.NAME_VALUE`, pattern) if err != nil { - c.Config().Log.Printf("%s: Query pattern %s: %v", c.String(), pattern, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: Query pattern %s: %v", c.String(), pattern, err)) return } c.SetActive() // Extract the subdomain names from the results - var names []string + names := stringset.New() for _, result := range results { - names = utils.UniqueAppend(names, strings.ToLower(utils.RemoveAsteriskLabel(result.Domain))) + names.Insert(utils.RemoveAsteriskLabel(result.Domain)) } - for _, name := range names { + for name := range names { c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, @@ -111,7 +114,7 @@ func (c *Crtsh) scrape(domain string) { url := c.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - c.Config().Log.Printf("%s: %s: %v", c.String(), url, err) + c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err)) return } diff --git a/services/sources/dnsdb.go b/services/sources/dnsdb.go index 851a5a446..cc6ed2322 100644 --- a/services/sources/dnsdb.go +++ b/services/sources/dnsdb.go @@ -16,6 +16,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -45,7 +46,7 @@ func (d *DNSDB) OnStart() error { d.API = d.Config().GetAPIKey(d.String()) if d.API == nil || d.API.Key == "" { - d.Config().Log.Printf("%s: API key data was not provided", d.String()) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", d.String())) } go d.processRequests() @@ -86,7 +87,7 @@ func (d *DNSDB) executeQuery(domain string) { url := d.restURL(domain) page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), url, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err)) return } @@ -101,13 +102,13 @@ func (d *DNSDB) restURL(domain string) string { } func (d *DNSDB) passiveDNSJSON(page, domain string) { - var unique []string re := d.Config().DomainRegex(domain) if re == nil { return } + unique := stringset.New() scanner := bufio.NewScanner(strings.NewReader(page)) for scanner.Scan() { // Get the next line of JSON @@ -124,11 +125,11 @@ func (d *DNSDB) passiveDNSJSON(page, domain string) { continue } if re.MatchString(j.Name) { - unique = utils.UniqueAppend(unique, j.Name) + unique.Insert(j.Name) } } - for _, name := range unique { + for name := range unique { d.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, @@ -142,18 +143,16 @@ func (d *DNSDB) scrape(domain string) { url := d.getURL(domain, domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), url, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err)) return } - var names []string - if f := d.followIndicies(page, domain); len(f) > 0 { - names = utils.UniqueAppend(names, f...) - } else if n := d.pullPageNames(page, domain); len(n) > 0 { - names = utils.UniqueAppend(names, n...) - } + names := stringset.New() + names.Union(d.followIndicies(page, domain)) + names.Union(d.pullPageNames(page, domain)) + // Share what has been discovered so far - for _, name := range names { + for name := range names { d.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, @@ -165,7 +164,7 @@ func (d *DNSDB) scrape(domain string) { t := time.NewTicker(d.RateLimit) defer t.Stop() loop: - for _, name := range names { + for name := range names { select { case <-d.Quit(): break loop @@ -177,11 +176,11 @@ loop: url = d.getURL(domain, name) another, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), url, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err)) continue } - for _, result := range d.pullPageNames(another, domain) { + for result := range d.pullPageNames(another, domain) { d.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: result, Domain: domain, @@ -210,8 +209,9 @@ func (d *DNSDB) getURL(domain, sub string) string { var dnsdbIndexRE = regexp.MustCompile(`([a-zA-Z0-9])`) -func (d *DNSDB) followIndicies(page, domain string) []string { - var indicies, unique []string +func (d *DNSDB) followIndicies(page, domain string) stringset.Set { + var indicies []string + unique := stringset.New() idx := dnsdbIndexRE.FindAllStringSubmatch(page, -1) if idx == nil { return unique @@ -231,22 +231,18 @@ func (d *DNSDB) followIndicies(page, domain string) []string { continue } - if names := d.pullPageNames(ipage, domain); len(names) > 0 { - unique = utils.UniqueAppend(unique, names...) - } + unique.Union(d.pullPageNames(ipage, domain)) time.Sleep(d.RateLimit) } return unique } -func (d *DNSDB) pullPageNames(page, domain string) []string { - var names []string +func (d *DNSDB) pullPageNames(page, domain string) stringset.Set { + names := stringset.New() if re := d.Config().DomainRegex(domain); re != nil { for _, name := range re.FindAllString(page, -1) { - if u := utils.NewUniqueElements(names, cleanName(name)); len(u) > 0 { - names = append(names, u...) - } + names.Insert(cleanName(name)) } } return names diff --git a/services/sources/dnsdumpster.go b/services/sources/dnsdumpster.go index 76cef078a..49622d62b 100644 --- a/services/sources/dnsdumpster.go +++ b/services/sources/dnsdumpster.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "io/ioutil" "net" "net/http" @@ -69,20 +70,20 @@ func (d *DNSDumpster) executeQuery(domain string) { u := "https://dnsdumpster.com/" page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), u, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), u, err)) return } token := d.getCSRFToken(page) if token == "" { - d.Config().Log.Printf("%s: %s: Failed to obtain the CSRF token", d.String(), u) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: Failed to obtain the CSRF token", d.String(), u)) return } d.SetActive() page, err = d.postForm(token, domain) if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), u, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), u, err)) return } @@ -120,7 +121,7 @@ func (d *DNSDumpster) postForm(token, domain string) (string, error) { req, err := http.NewRequest("POST", "https://dnsdumpster.com/", strings.NewReader(params.Encode())) if err != nil { - d.Config().Log.Printf("%s: Failed to setup the POST request: %v", d.String(), err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: Failed to setup the POST request: %v", d.String(), err)) return "", err } // The CSRF token needs to be sent as a cookie @@ -140,7 +141,7 @@ func (d *DNSDumpster) postForm(token, domain string) (string, error) { resp, err := client.Do(req) if err != nil { - d.Config().Log.Printf("%s: The POST request failed: %v", d.String(), err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: The POST request failed: %v", d.String(), err)) return "", err } // Now, grab the entire page diff --git a/services/sources/dnstable.go b/services/sources/dnstable.go index 65e0cb91f..4d4b711d9 100644 --- a/services/sources/dnstable.go +++ b/services/sources/dnstable.go @@ -63,7 +63,7 @@ func (d *DNSTable) executeQuery(domain string) { url := d.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), url, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err)) return } diff --git a/services/sources/dogpile.go b/services/sources/dogpile.go index 9681ceccf..388889443 100644 --- a/services/sources/dogpile.go +++ b/services/sources/dogpile.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "strconv" "time" @@ -81,7 +82,7 @@ func (d *Dogpile) executeQuery(domain string) { u := d.urlByPageNum(domain, i) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - d.Config().Log.Printf("%s: %s: %v", d.String(), u, err) + d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), u, err)) return } diff --git a/services/sources/entrust.go b/services/sources/entrust.go index 004197d78..84d2de413 100644 --- a/services/sources/entrust.go +++ b/services/sources/entrust.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "regexp" "strings" @@ -65,7 +66,7 @@ func (e *Entrust) executeQuery(domain string) { u := e.getURL(domain) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - e.Config().Log.Printf("%s: %s: %v", e.String(), u, err) + e.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", e.String(), u, err)) return } content := strings.Replace(page, "u003d", " ", -1) diff --git a/services/sources/exalead.go b/services/sources/exalead.go index 145fce826..5ac636ec7 100644 --- a/services/sources/exalead.go +++ b/services/sources/exalead.go @@ -63,7 +63,7 @@ func (e *Exalead) executeQuery(domain string) { url := e.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - e.Config().Log.Printf("%s: %s: %v", e.String(), url, err) + e.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", e.String(), url, err)) return } diff --git a/services/sources/google.go b/services/sources/google.go index ffd07b1d2..60b7b12a4 100644 --- a/services/sources/google.go +++ b/services/sources/google.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "strconv" "time" @@ -81,7 +82,7 @@ func (g *Google) executeQuery(domain string) { u := g.urlByPageNum(domain, i) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - g.Config().Log.Printf("%s: %s: %v", g.String(), u, err) + g.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", g.String(), u, err)) return } diff --git a/services/sources/googlect.go b/services/sources/googlect.go index 8e0e83391..0681d97ca 100644 --- a/services/sources/googlect.go +++ b/services/sources/googlect.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net/url" "regexp" "time" @@ -81,7 +82,7 @@ func (g *GoogleCT) executeDNSQuery(domain string) { } page, err := utils.RequestWebPage(u, nil, headers, "", "") if err != nil { - g.Config().Log.Printf("%s: %s: %v", g.String(), u, err) + g.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", g.String(), u, err)) break } diff --git a/services/sources/hackerone.go b/services/sources/hackerone.go index f925ba59d..695949f60 100644 --- a/services/sources/hackerone.go +++ b/services/sources/hackerone.go @@ -64,7 +64,7 @@ func (h *HackerOne) executeDNSQuery(domain string) { url := h.getDNSURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - h.Config().Log.Printf("%s: %s: %v", h.String(), url, err) + h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", h.String(), url, err)) return } diff --git a/services/sources/hackerone_test.go b/services/sources/hackerone_test.go new file mode 100644 index 000000000..e1da64ea3 --- /dev/null +++ b/services/sources/hackerone_test.go @@ -0,0 +1,31 @@ +package sources + +import ( + "testing" + + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" +) + +func TestHackerone(t *testing.T) { + if *networkTest == false || *configPath == "" { + return + } + + domainTest = "twitter.com" + + cfg := setupConfig(domainTest) + + bus, out := setupEventBus(requests.NewNameTopic) + defer bus.Stop() + + pool := resolvers.NewResolverPool(nil) + defer pool.Stop() + + srv := NewHackerOne(cfg, bus, pool) + + result := testService(srv, out) + if result < expectedTest { + t.Errorf("Found %d names, expected at least %d instead", result, expectedTest) + } +} diff --git a/services/sources/hackertarget.go b/services/sources/hackertarget.go index d11676e65..7a9cfd042 100644 --- a/services/sources/hackertarget.go +++ b/services/sources/hackertarget.go @@ -14,6 +14,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -68,7 +69,7 @@ func (h *HackerTarget) executeDNSQuery(domain string) { url := h.getDNSURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - h.Config().Log.Printf("%s: %s: %v", h.String(), url, err) + h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", h.String(), url, err)) return } @@ -96,19 +97,21 @@ func (h *HackerTarget) executeASNQuery(addr string) { url := h.getASNURL(addr) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - h.Config().Log.Printf("%s: %s: %v", h.String(), url, err) + h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", h.String(), url, err)) return } fields := strings.Split(page, ",") if len(fields) < 4 { - h.Config().Log.Printf("%s: %s: Failed to parse the response", h.String(), url) + h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: Failed to parse the response", h.String(), url)) return } asn, err := strconv.Atoi(strings.Trim(fields[1], "\"")) if err != nil { - h.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", h.String(), url, err) + h.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", h.String(), url, err), + ) return } @@ -117,7 +120,7 @@ func (h *HackerTarget) executeASNQuery(addr string) { Prefix: strings.Trim(fields[2], "\""), AllocationDate: time.Now(), Description: strings.Trim(fields[3], "\""), - Netblocks: []string{strings.Trim(fields[2], "\"")}, + Netblocks: stringset.New(strings.Trim(fields[2], "\"")), Tag: h.SourceType, Source: h.String(), }) diff --git a/services/sources/ipv4info.go b/services/sources/ipv4info.go index 665f3bed1..5754b7a0a 100644 --- a/services/sources/ipv4info.go +++ b/services/sources/ipv4info.go @@ -68,7 +68,7 @@ func (i *IPv4Info) executeQuery(domain string) { url := i.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - i.Config().Log.Printf("%s: %s: %v", i.String(), url, err) + i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err)) return } @@ -77,7 +77,7 @@ func (i *IPv4Info) executeQuery(domain string) { url = i.ipSubmatch(page, domain) page, err = utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - i.Config().Log.Printf("%s: %s: %v", i.String(), url, err) + i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err)) return } @@ -86,7 +86,7 @@ func (i *IPv4Info) executeQuery(domain string) { url = i.domainSubmatch(page, domain) page, err = utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - i.Config().Log.Printf("%s: %s: %v", i.String(), url, err) + i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err)) return } @@ -95,7 +95,7 @@ func (i *IPv4Info) executeQuery(domain string) { url = i.subdomainSubmatch(page, domain) page, err = utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - i.Config().Log.Printf("%s: %s: %v", i.String(), url, err) + i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err)) return } diff --git a/services/sources/locarchive.go b/services/sources/locarchive.go index da6983185..e959a597b 100644 --- a/services/sources/locarchive.go +++ b/services/sources/locarchive.go @@ -4,6 +4,8 @@ package sources import ( + "fmt" + "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" @@ -70,7 +72,7 @@ func (l *LoCArchive) executeQuery(sn, domain string) { names, err := crawl(l, l.baseURL, l.domain, sn, domain) if err != nil { - l.Config().Log.Printf("%s: %v", l.String(), err) + l.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", l.String(), err)) return } diff --git a/services/sources/mnemonic.go b/services/sources/mnemonic.go index 3b440cc06..d75aecc9c 100644 --- a/services/sources/mnemonic.go +++ b/services/sources/mnemonic.go @@ -14,6 +14,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -61,12 +62,12 @@ func (m *Mnemonic) executeDNSQuery(domain string) { url := m.getDNSURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - m.Config().Log.Printf("%s: %s: %v", m.String(), url, err) + m.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", m.String(), url, err)) return } - var ips []string - var names []string + ips := stringset.New() + names := stringset.New() scanner := bufio.NewScanner(strings.NewReader(page)) for scanner.Scan() { // Get the next line of JSON @@ -85,12 +86,12 @@ func (m *Mnemonic) executeDNSQuery(domain string) { } if (j.Type == "a" || j.Type == "aaaa") && m.Config().IsDomainInScope(j.Query) { - ips = utils.UniqueAppend(ips, j.Answer) - names = utils.UniqueAppend(names, j.Query) + ips.Insert(j.Answer) + names.Insert(j.Query) } } - for _, name := range names { + for name := range names { m.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, Domain: domain, @@ -99,7 +100,7 @@ func (m *Mnemonic) executeDNSQuery(domain string) { }) } - for _, ip := range ips { + for ip := range ips { // Inform the Address Service of this finding m.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{ Address: ip, diff --git a/services/sources/netcraft.go b/services/sources/netcraft.go index 4229c946c..df65ce2ee 100644 --- a/services/sources/netcraft.go +++ b/services/sources/netcraft.go @@ -63,7 +63,7 @@ func (n *Netcraft) executeQuery(domain string) { url := n.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - n.Config().Log.Printf("%s: %s, %v", n.String(), url, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), url, err)) return } diff --git a/services/sources/networksdb.go b/services/sources/networksdb.go index 57ea41f69..f12059042 100644 --- a/services/sources/networksdb.go +++ b/services/sources/networksdb.go @@ -5,6 +5,7 @@ package sources import ( "encoding/json" + "fmt" "net" "net/url" "regexp" @@ -17,6 +18,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -62,7 +64,7 @@ func (n *NetworksDB) OnStart() error { n.API = n.Config().GetAPIKey(n.String()) if n.API == nil || n.API.Key == "" { - n.Config().Log.Printf("%s: API key data was not provided", n.String()) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", n.String())) n.SourceType = requests.SCRAPE n.hasAPIKey = false } @@ -99,7 +101,7 @@ loop: if req.Address != "" { n.executeASNAddrQuery(req.Address) } else { - n.executeASNQuery(req.ASN, "", []string{}) + n.executeASNQuery(req.ASN, "", stringset.New()) } } last = time.Now() @@ -113,13 +115,15 @@ func (n *NetworksDB) executeASNAddrQuery(addr string) { u := n.getIPURL(addr) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return } matches := networksdbOrgLinkRE.FindStringSubmatch(page) if matches == nil || len(matches) < 2 { - n.Config().Log.Printf("%s: %s: Failed to extract the organization info href", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to extract the organization info href", n.String(), u), + ) return } @@ -128,26 +132,30 @@ func (n *NetworksDB) executeASNAddrQuery(addr string) { u = networksdbBaseURL + matches[1] page, err = utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return } - var netblocks []string + netblocks := stringset.New() for _, match := range networksdbCIDRRE.FindAllStringSubmatch(page, -1) { if len(match) >= 2 { - netblocks = utils.UniqueAppend(netblocks, strings.TrimSpace(match[1])) + netblocks.Insert(strings.TrimSpace(match[1])) } } matches = networksdbASNRE.FindStringSubmatch(page) if matches == nil || len(matches) < 2 { - n.Config().Log.Printf("%s: %s: The regular expression failed to extract the ASN", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The regular expression failed to extract the ASN", n.String(), u), + ) return } asn, err := strconv.Atoi(strings.TrimSpace(matches[1])) if err != nil { - n.Config().Log.Printf("%s: %s: Failed to extract a valid ASN", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to extract a valid ASN", n.String(), u), + ) return } @@ -160,32 +168,36 @@ func (n *NetworksDB) getIPURL(addr string) string { return networksdbBaseURL + "/ip/" + addr } -func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) { +func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks stringset.Set) { n.SetActive() u := n.getASNURL(asn) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return } matches := networksdbASNameRE.FindStringSubmatch(page) if matches == nil || len(matches) < 2 { - n.Config().Log.Printf("%s: The regular expression failed to extract the AS name", n.String()) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: The regular expression failed to extract the AS name", n.String()), + ) return } name := strings.TrimSpace(matches[1]) matches = networksdbCCRE.FindStringSubmatch(page) if matches == nil || len(matches) < 2 { - n.Config().Log.Printf("%s: The regular expression failed to extract the country code", n.String()) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: The regular expression failed to extract the country code", n.String()), + ) return } cc := strings.TrimSpace(matches[1]) for _, match := range networksdbCIDRRE.FindAllStringSubmatch(page, -1) { if len(match) >= 2 { - netblocks = utils.UniqueAppend(netblocks, strings.TrimSpace(match[1])) + netblocks.Insert(strings.TrimSpace(match[1])) } } @@ -193,7 +205,7 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) { if addr != "" { ip := net.ParseIP(addr) - for _, cidr := range netblocks { + for cidr := range netblocks { if _, ipnet, err := net.ParseCIDR(cidr); err == nil && ipnet.Contains(ip) { prefix = cidr break @@ -201,7 +213,7 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) { } } if prefix == "" && len(netblocks) > 0 { - prefix = netblocks[0] + prefix = netblocks.Slice()[0] // TODO order may matter here :shrug: } n.Bus().Publish(requests.NewASNTopic, &requests.ASNRequest{ @@ -223,29 +235,35 @@ func (n *NetworksDB) getASNURL(asn int) string { func (n *NetworksDB) executeAPIASNAddrQuery(addr string) { _, id := n.apiIPQuery(addr) if id == "" { - n.Config().Log.Printf("%s: %s: Failed to obtain IP address information", n.String(), addr) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to obtain IP address information", n.String(), addr), + ) return } time.Sleep(n.RateLimit) asns := n.apiOrgInfoQuery(id) if len(asns) == 0 { - n.Config().Log.Printf("%s: %s: Failed to obtain ASNs associated with the organization", n.String(), id) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to obtain ASNs associated with the organization", n.String(), id), + ) return } var asn int - var cidrs []string + cidrs := stringset.New() ip := net.ParseIP(addr) loop: for _, a := range asns { time.Sleep(n.RateLimit) cidrs = n.apiNetblocksQuery(a) if len(cidrs) == 0 { - n.Config().Log.Printf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), a) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), a), + ) } - for _, cidr := range cidrs { + for cidr := range cidrs { if _, ipnet, err := net.ParseCIDR(cidr); err == nil { if ipnet.Contains(ip) { asn = a @@ -256,17 +274,21 @@ loop: } if asn == 0 { - n.Config().Log.Printf("%s: %s: Failed to obtain the ASN associated with the IP address", n.String(), addr) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to obtain the ASN associated with the IP address", n.String(), addr), + ) return } n.executeAPIASNQuery(asn, addr, cidrs) } -func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string) { - if netblocks == nil { - netblocks = n.apiNetblocksQuery(asn) +func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks stringset.Set) { + if len(netblocks) == 0 { + netblocks.Union(n.apiNetblocksQuery(asn)) if len(netblocks) == 0 { - n.Config().Log.Printf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), asn) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), asn), + ) return } } @@ -274,7 +296,7 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string var prefix string if addr != "" { ip := net.ParseIP(addr) - for _, cidr := range netblocks { + for cidr := range netblocks { if _, ipnet, err := net.ParseCIDR(cidr); err == nil && ipnet.Contains(ip) { prefix = cidr break @@ -282,13 +304,15 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string } } if prefix == "" { - prefix = netblocks[0] + prefix = netblocks.Slice()[0] } time.Sleep(n.RateLimit) req := n.apiASNInfoQuery(asn) if req == nil { - n.Config().Log.Printf("%s: %d: Failed to obtain ASN information", n.String(), asn) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %d: Failed to obtain ASN information", n.String(), asn), + ) return } @@ -307,7 +331,7 @@ func (n *NetworksDB) apiIPQuery(addr string) (string, string) { body := strings.NewReader(params.Encode()) page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return "", "" } @@ -324,13 +348,15 @@ func (n *NetworksDB) apiIPQuery(addr string) (string, string) { } `json:"results"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return "", "" } else if m.Error != "" { - n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error)) return "", "" } else if m.Total == 0 || len(m.Results) == 0 { - n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u), + ) return "", "" } @@ -348,7 +374,7 @@ func (n *NetworksDB) apiOrgInfoQuery(id string) []int { body := strings.NewReader(params.Encode()) page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return []int{} } @@ -360,13 +386,15 @@ func (n *NetworksDB) apiOrgInfoQuery(id string) []int { } `json:"results"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return []int{} } else if m.Error != "" { - n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error)) return []int{} } else if m.Total == 0 || len(m.Results[0].ASNs) == 0 { - n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u), + ) return []int{} } @@ -384,7 +412,7 @@ func (n *NetworksDB) apiASNInfoQuery(asn int) *requests.ASNRequest { body := strings.NewReader(params.Encode()) page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return nil } @@ -400,13 +428,15 @@ func (n *NetworksDB) apiASNInfoQuery(asn int) *requests.ASNRequest { } `json:"results"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return nil } else if m.Error != "" { - n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error)) return nil } else if m.Total == 0 || len(m.Results) == 0 { - n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u), + ) return nil } @@ -423,8 +453,8 @@ func (n *NetworksDB) getAPIASNInfoURL() string { return networksdbBaseURL + networksdbAPIPATH + "/as/info" } -func (n *NetworksDB) apiNetblocksQuery(asn int) []string { - var netblocks []string +func (n *NetworksDB) apiNetblocksQuery(asn int) stringset.Set { + netblocks := stringset.New() n.SetActive() u := n.getAPINetblocksURL() @@ -432,7 +462,7 @@ func (n *NetworksDB) apiNetblocksQuery(asn int) []string { body := strings.NewReader(params.Encode()) page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "") if err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return netblocks } @@ -444,18 +474,20 @@ func (n *NetworksDB) apiNetblocksQuery(asn int) []string { } `json:"results"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - n.Config().Log.Printf("%s: %s: %v", n.String(), u, err) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err)) return netblocks } else if m.Error != "" { - n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error) + n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error)) return netblocks } else if m.Total == 0 || len(m.Results) == 0 { - n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u) + n.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u), + ) return netblocks } for _, block := range m.Results { - netblocks = utils.UniqueAppend(netblocks, block.CIDR) + netblocks.Insert(block.CIDR) } return netblocks } diff --git a/services/sources/openukarchive.go b/services/sources/openukarchive.go index 25c312fb4..ed8fb9496 100644 --- a/services/sources/openukarchive.go +++ b/services/sources/openukarchive.go @@ -4,6 +4,8 @@ package sources import ( + "fmt" + "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" @@ -67,7 +69,7 @@ func (o *OpenUKArchive) executeQuery(sn, domain string) { names, err := crawl(o, o.baseURL, o.domain, sn, domain) if err != nil { - o.Config().Log.Printf("%s: %v", o.String(), err) + o.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", o.String(), err)) return } diff --git a/services/sources/passivetotal.go b/services/sources/passivetotal.go index 8f5632a5c..03f36d48c 100644 --- a/services/sources/passivetotal.go +++ b/services/sources/passivetotal.go @@ -5,6 +5,7 @@ package sources import ( "encoding/json" + "fmt" "time" "github.com/OWASP/Amass/config" @@ -41,7 +42,7 @@ func (pt *PassiveTotal) OnStart() error { pt.API = pt.Config().GetAPIKey(pt.String()) if pt.API == nil || pt.API.Username == "" || pt.API.Key == "" { - pt.Config().Log.Printf("%s: API key data was not provided", pt.String()) + pt.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", pt.String())) } go pt.processRequests() @@ -86,7 +87,7 @@ func (pt *PassiveTotal) executeQuery(domain string) { headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, pt.API.Username, pt.API.Key) if err != nil { - pt.Config().Log.Printf("%s: %s: %v", pt.String(), url, err) + pt.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", pt.String(), url, err)) return } // Extract the subdomain names from the REST API results diff --git a/services/sources/passivetotal_test.go b/services/sources/passivetotal_test.go new file mode 100644 index 000000000..9ada6f78f --- /dev/null +++ b/services/sources/passivetotal_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" + + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" +) + +func TestPassiveTotal(t *testing.T) { + if *networkTest == false || *configPath == "" { + return + } + + cfg := setupConfig(domainTest) + api := cfg.GetAPIKey("passivetotal") + if api == nil || api.Username == "" || api.Key == "" { + t.Errorf("API key data was not provided") + return + } + + bus, out := setupEventBus(requests.NewNameTopic) + defer bus.Stop() + + pool := resolvers.NewResolverPool(nil) + defer pool.Stop() + + srv := NewPassiveTotal(cfg, bus, pool) + + result := testService(srv, out) + if result < expectedTest { + t.Errorf("Found %d names, expected at least %d instead", result, expectedTest) + } +} diff --git a/services/sources/ptrarchive.go b/services/sources/ptrarchive.go index 32a99b3c5..a73e53ccf 100644 --- a/services/sources/ptrarchive.go +++ b/services/sources/ptrarchive.go @@ -63,7 +63,7 @@ func (p *PTRArchive) executeQuery(domain string) { url := p.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - p.Config().Log.Printf("%s: %s: %v", p.String(), url, err) + p.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", p.String(), url, err)) return } diff --git a/services/sources/radb.go b/services/sources/radb.go index 5c3089406..f6905d492 100644 --- a/services/sources/radb.go +++ b/services/sources/radb.go @@ -18,6 +18,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -120,7 +121,7 @@ func (r *RADb) executeASNAddrQuery(addr string) { headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return } @@ -134,10 +135,12 @@ func (r *RADb) executeASNAddrQuery(addr string) { } `json:"cidr0_cidrs"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return } else if m.ClassName != "ip network" || len(m.CIDRs) == 0 { - r.Config().Log.Printf("%s: %s: The request returned zero results", r.String(), url) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The request returned zero results", r.String(), url), + ) return } @@ -176,7 +179,7 @@ func (r *RADb) executeASNQuery(asn int, prefix string) { headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return } @@ -189,10 +192,12 @@ func (r *RADb) executeASNQuery(asn int, prefix string) { } } if err := json.Unmarshal([]byte(page), &m); err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return } else if m.ClassName != "autnum" { - r.Config().Log.Printf("%s: %s: The query returned incorrect results", r.String(), url) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The query returned incorrect results", r.String(), url), + ) return } @@ -215,20 +220,22 @@ func (r *RADb) executeASNQuery(asn int, prefix string) { r.SetActive() time.Sleep(r.RateLimit) - var blocks []string + blocks := stringset.New() if prefix != "" { - blocks = []string{prefix} + blocks.Insert(prefix) } - blocks = utils.UniqueAppend(blocks, r.netblocks(asn)...) + blocks.Union(r.netblocks(asn)) if len(blocks) == 0 { - r.Config().Log.Printf("%s: %s: The query returned zero netblocks", r.String(), url) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The query returned zero netblocks", r.String(), url), + ) return } r.Bus().Publish(requests.NewASNTopic, &requests.ASNRequest{ ASN: asn, - Prefix: blocks[0], + Prefix: prefix, AllocationDate: at, Description: m.Description, Netblocks: blocks, @@ -243,15 +250,15 @@ func (r *RADb) getASNURL(registry, asn string) string { return fmt.Sprintf(format, asn) } -func (r *RADb) netblocks(asn int) []string { - var netblocks []string +func (r *RADb) netblocks(asn int) stringset.Set { + netblocks := stringset.New() r.SetActive() url := r.getNetblocksURL(strconv.Itoa(asn)) headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return netblocks } @@ -267,7 +274,7 @@ func (r *RADb) netblocks(asn int) []string { } `json:"arin_originas0_networkSearchResults"` } if err := json.Unmarshal([]byte(page), &m); err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return netblocks } @@ -289,13 +296,15 @@ func (r *RADb) netblocks(asn int) []string { if prefix != "" { l := strconv.Itoa(cidr.Length) - netblocks = utils.UniqueAppend(netblocks, prefix+"/"+l) + netblocks.Insert(prefix + "/" + l) } } } if len(netblocks) == 0 { - r.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn), + ) } return netblocks } @@ -311,13 +320,15 @@ func (r *RADb) ipToASN(cidr string) int { if r.addr == "" { answers, err := r.Pool().Resolve(radbWhoisURL, "A", resolvers.PriorityHigh) if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), radbWhoisURL, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), radbWhoisURL, err)) return 0 } ip := answers[0].Data if ip == "" { - r.Config().Log.Printf("%s: Failed to resolve %s", r.String(), radbWhoisURL) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to resolve %s", r.String(), radbWhoisURL), + ) return 0 } r.addr = ip @@ -329,7 +340,7 @@ func (r *RADb) ipToASN(cidr string) int { d := net.Dialer{} conn, err := d.DialContext(ctx, "tcp", r.addr+":43") if err != nil { - r.Config().Log.Printf("%s: %v", r.String(), err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", r.String(), err)) return 0 } defer conn.Close() diff --git a/services/sources/riddler.go b/services/sources/riddler.go index b417db509..0e8cd1284 100644 --- a/services/sources/riddler.go +++ b/services/sources/riddler.go @@ -63,7 +63,7 @@ func (r *Riddler) executeQuery(domain string) { url := r.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return } diff --git a/services/sources/robtex.go b/services/sources/robtex.go index 7ee43f3a6..b88641119 100644 --- a/services/sources/robtex.go +++ b/services/sources/robtex.go @@ -6,6 +6,7 @@ package sources import ( "bufio" "encoding/json" + "fmt" "net" "strconv" "strings" @@ -16,6 +17,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -90,8 +92,6 @@ loop: } func (r *Robtex) executeDNSQuery(domain string) { - var ips []string - re := r.Config().DomainRegex(domain) if re == nil { return @@ -101,13 +101,14 @@ func (r *Robtex) executeDNSQuery(domain string) { url := "https://freeapi.robtex.com/pdns/forward/" + domain page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return } + ips := stringset.New() for _, line := range r.parseDNSJSON(page) { if line.Type == "A" { - ips = utils.UniqueAppend(ips, line.Data) + ips.Insert(line.Data) // Inform the Address Service of this finding r.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{ Address: line.Data, @@ -118,11 +119,11 @@ func (r *Robtex) executeDNSQuery(domain string) { } } - var names []string + names := stringset.New() t := time.NewTicker(500 * time.Millisecond) defer t.Stop() loop: - for _, ip := range ips { + for ip := range ips { r.SetActive() select { @@ -132,17 +133,17 @@ loop: url = "https://freeapi.robtex.com/pdns/reverse/" + ip pdns, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) continue } for _, line := range r.parseDNSJSON(pdns) { - names = utils.UniqueAppend(names, line.Name) + names.Insert(line.Name) } } } - for _, name := range names { + for name := range names { if r.Config().IsDomainInScope(name) { r.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, @@ -182,7 +183,7 @@ func (r *Robtex) executeASNQuery(asn int) { return } - _, ipnet, err := net.ParseCIDR(blocks[0]) + _, ipnet, err := net.ParseCIDR(blocks.Slice()[0]) if err != nil { return } @@ -194,7 +195,7 @@ func (r *Robtex) executeASNQuery(asn int) { return } - req.Netblocks = utils.UniqueAppend(req.Netblocks, blocks...) + req.Netblocks.Union(blocks) r.Bus().Publish(requests.NewASNTopic, req) } @@ -207,7 +208,7 @@ func (r *Robtex) executeASNAddrQuery(addr string) { r.SetActive() time.Sleep(r.RateLimit) - req.Netblocks = utils.UniqueAppend(req.Netblocks, r.netblocks(req.ASN)...) + req.Netblocks.Union(r.netblocks(req.ASN)) r.Bus().Publish(requests.NewASNTopic, req) } @@ -220,7 +221,7 @@ func (r *Robtex) origin(addr string) *requests.ASNRequest { url := "https://freeapi.robtex.com/ipquery/" + addr page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return nil } // Extract the network information @@ -293,7 +294,9 @@ func (r *Robtex) origin(addr string) *requests.ASNRequest { } if ipinfo.ASN == 0 { - r.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", r.String(), url, err), + ) return nil } @@ -309,20 +312,20 @@ func (r *Robtex) origin(addr string) *requests.ASNRequest { ASN: ipinfo.ASN, Prefix: ipinfo.Prefix, Description: desc, - Netblocks: []string{ipinfo.Prefix}, + Netblocks: stringset.New(ipinfo.Prefix), Tag: r.SourceType, Source: r.String(), } } -func (r *Robtex) netblocks(asn int) []string { - var netblocks []string +func (r *Robtex) netblocks(asn int) stringset.Set { + netblocks := stringset.New() r.SetActive() url := "https://freeapi.robtex.com/asquery/" + strconv.Itoa(asn) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - r.Config().Log.Printf("%s: %s: %v", r.String(), url, err) + r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err)) return netblocks } // Extract the network information @@ -337,11 +340,13 @@ func (r *Robtex) netblocks(asn int) []string { } for _, net := range n.Networks { - netblocks = utils.UniqueAppend(netblocks, net.CIDR) + netblocks.Insert(net.CIDR) } if len(netblocks) == 0 { - r.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn) + r.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn), + ) } return netblocks } diff --git a/services/sources/securitytrails.go b/services/sources/securitytrails.go index 5ad64733a..ed67f4dfc 100644 --- a/services/sources/securitytrails.go +++ b/services/sources/securitytrails.go @@ -43,7 +43,9 @@ func (st *SecurityTrails) OnStart() error { st.API = st.Config().GetAPIKey(st.String()) if st.API == nil || st.API.Key == "" { - st.Config().Log.Printf("%s: API key data was not provided", st.String()) + st.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: API key data was not provided", st.String()), + ) } go st.processRequests() @@ -88,7 +90,7 @@ func (st *SecurityTrails) executeQuery(domain string) { st.SetActive() page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - st.Config().Log.Printf("%s: %s: %v", st.String(), url, err) + st.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", st.String(), url, err)) return } // Extract the subdomain names from the REST API results diff --git a/services/sources/securitytrails_test.go b/services/sources/securitytrails_test.go new file mode 100644 index 000000000..d5d71798d --- /dev/null +++ b/services/sources/securitytrails_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" + + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" +) + +func TestSecurityTrails(t *testing.T) { + if *networkTest == false || *configPath == "" { + return + } + + cfg := setupConfig(domainTest) + api := cfg.GetAPIKey("securitytrails") + if api == nil || api.Key == "" { + t.Errorf("API key data was not provided") + return + } + + bus, out := setupEventBus(requests.NewNameTopic) + defer bus.Stop() + + pool := resolvers.NewResolverPool(nil) + defer pool.Stop() + + srv := NewSecurityTrails(cfg, bus, pool) + + result := testService(srv, out) + if result < expectedTest { + t.Errorf("Found %d names, expected at least %d instead", result, expectedTest) + } +} diff --git a/services/sources/shadowserver.go b/services/sources/shadowserver.go index d340af5bd..7a654c41c 100644 --- a/services/sources/shadowserver.go +++ b/services/sources/shadowserver.go @@ -17,6 +17,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -98,12 +99,12 @@ func (s *ShadowServer) executeASNQuery(asn int) { } time.Sleep(s.RateLimit) - req := s.origin(strings.Trim(blocks[0], "/")) + req := s.origin(strings.Trim(blocks.Slice()[0], "/")) if req == nil { return } - req.Netblocks = utils.UniqueAppend(req.Netblocks, blocks...) + req.Netblocks.Union(blocks) s.Bus().Publish(requests.NewASNTopic, req) } @@ -115,7 +116,7 @@ func (s *ShadowServer) executeASNAddrQuery(addr string) { } time.Sleep(s.RateLimit) - req.Netblocks = utils.UniqueAppend(req.Netblocks, s.netblocks(req.ASN)...) + req.Netblocks.Union(s.netblocks(req.ASN)) s.Bus().Publish(requests.NewASNTopic, req) } @@ -127,19 +128,25 @@ func (s *ShadowServer) origin(addr string) *requests.ASNRequest { answers, err := s.Pool().Resolve(name, "TXT", resolvers.PriorityHigh) if err != nil { - s.Config().Log.Printf("%s: %s: DNS TXT record query error: %v", s.String(), name, err) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: DNS TXT record query error: %v", s.String(), name, err), + ) return nil } fields := strings.Split(strings.Trim(answers[0].Data, "\""), " | ") if len(fields) < 5 { - s.Config().Log.Printf("%s: %s: Failed to parse the origin response", s.String(), name) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response", s.String(), name), + ) return nil } asn, err := strconv.Atoi(strings.TrimSpace(fields[0])) if err != nil { - s.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", s.String(), name, err) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", s.String(), name, err), + ) return nil } @@ -148,25 +155,29 @@ func (s *ShadowServer) origin(addr string) *requests.ASNRequest { Prefix: strings.TrimSpace(fields[1]), CC: strings.TrimSpace(fields[3]), Description: strings.TrimSpace(fields[2]) + " - " + strings.TrimSpace(fields[4]), - Netblocks: []string{strings.TrimSpace(fields[1])}, + Netblocks: stringset.New(strings.TrimSpace(fields[1])), Tag: s.SourceType, Source: s.String(), } } -func (s *ShadowServer) netblocks(asn int) []string { - var netblocks []string +func (s *ShadowServer) netblocks(asn int) stringset.Set { + netblocks := stringset.New() if s.addr == "" { answers, err := s.Pool().Resolve(ShadowServerWhoisURL, "A", resolvers.PriorityHigh) if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), ShadowServerWhoisURL, err) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: %v", s.String(), ShadowServerWhoisURL, err), + ) return netblocks } ip := answers[0].Data if ip == "" { - s.Config().Log.Printf("%s: Failed to resolve %s", s.String(), ShadowServerWhoisURL) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to resolve %s", s.String(), ShadowServerWhoisURL), + ) return netblocks } s.addr = ip @@ -178,7 +189,7 @@ func (s *ShadowServer) netblocks(asn int) []string { d := net.Dialer{} conn, err := d.DialContext(ctx, "tcp", s.addr+":43") if err != nil { - s.Config().Log.Printf("%s: %v", s.String(), err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", s.String(), err)) return netblocks } defer conn.Close() @@ -190,12 +201,14 @@ func (s *ShadowServer) netblocks(asn int) []string { line := scanner.Text() if err := scanner.Err(); err == nil { - netblocks = utils.UniqueAppend(netblocks, strings.TrimSpace(line)) + netblocks.Insert(strings.TrimSpace(line)) } } if len(netblocks) == 0 { - s.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", s.String(), asn) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to acquire netblocks for ASN %d", s.String(), asn), + ) } return netblocks } diff --git a/services/sources/shodan.go b/services/sources/shodan.go index 3f39a696a..70d17dc30 100644 --- a/services/sources/shodan.go +++ b/services/sources/shodan.go @@ -42,7 +42,9 @@ func (s *Shodan) OnStart() error { s.API = s.Config().GetAPIKey(s.String()) if s.API == nil || s.API.Key == "" { - s.Config().Log.Printf("%s: API key data was not provided", s.String()) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: API key data was not provided", s.String()), + ) } go s.processRequests() @@ -83,7 +85,7 @@ func (s *Shodan) executeQuery(domain string) { headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), url, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err)) return } // Extract the subdomain names from the REST API results diff --git a/services/sources/shodan_test.go b/services/sources/shodan_test.go new file mode 100644 index 000000000..bdea99946 --- /dev/null +++ b/services/sources/shodan_test.go @@ -0,0 +1,34 @@ +package sources + +import ( + "testing" + + "github.com/OWASP/Amass/requests" + "github.com/OWASP/Amass/resolvers" +) + +func TestShodan(t *testing.T) { + if *networkTest == false || *configPath == "" { + return + } + + cfg := setupConfig(domainTest) + api := cfg.GetAPIKey("shodan") + if api == nil || api.Key == "" { + t.Errorf("API key data was not provided") + return + } + + bus, out := setupEventBus(requests.NewNameTopic) + defer bus.Stop() + + pool := resolvers.NewResolverPool(nil) + defer pool.Stop() + + srv := NewShodan(cfg, bus, pool) + + result := testService(srv, out) + if result < expectedTest { + t.Errorf("Found %d names, expected at least %d instead", result, expectedTest) + } +} diff --git a/services/sources/sitedossier.go b/services/sources/sitedossier.go index 8beab497d..32e1aa803 100644 --- a/services/sources/sitedossier.go +++ b/services/sources/sitedossier.go @@ -63,7 +63,7 @@ func (s *SiteDossier) executeQuery(domain string) { url := s.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), url, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err)) return } diff --git a/services/sources/sources.go b/services/sources/sources.go index 76dc09d91..6dd8d250c 100644 --- a/services/sources/sources.go +++ b/services/sources/sources.go @@ -14,6 +14,7 @@ import ( eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" "github.com/PuerkitoBio/goquery" "github.com/geziyor/geziyor" @@ -103,14 +104,14 @@ func cleanName(name string) string { } func crawl(service services.Service, baseURL, baseDomain, subdomain, domain string) ([]string, error) { - var results []string + results := stringset.New() maxCrawlSem.Acquire(1) defer maxCrawlSem.Release(1) re := service.Config().DomainRegex(domain) if re == nil { - return results, fmt.Errorf("crawler error: Failed to obtain regex object for: %s", domain) + return results.Slice(), fmt.Errorf("crawler error: Failed to obtain regex object for: %s", domain) } start := fmt.Sprintf("%s/%s/%s", baseURL, strconv.Itoa(time.Now().Year()), subdomain) @@ -128,12 +129,12 @@ func crawl(service services.Service, baseURL, baseDomain, subdomain, domain stri r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) { if href, ok := s.Attr("href"); ok { if sub := re.FindString(r.JoinURL(href)); sub != "" { - results = utils.UniqueAppend(results, cleanName(sub)) + results.Insert(cleanName(sub)) } } }) }, }).Start() - return results, nil + return results.Slice(), nil } diff --git a/services/sources/spyse.go b/services/sources/spyse.go index a8e997e6a..1e6517ce2 100644 --- a/services/sources/spyse.go +++ b/services/sources/spyse.go @@ -37,7 +37,9 @@ func (s *Spyse) OnStart() error { s.API = s.Config().GetAPIKey(s.String()) if s.API == nil || s.API.Key == "" { - s.Config().Log.Printf("%s: API key data was not provided", s.String()) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: API key data was not provided", s.String()), + ) } go s.processRequests() @@ -83,7 +85,7 @@ func (s *Spyse) subdomainQueryAPI(domain string, page int) (int, error) { u := s.getAPIURL(domain, page) response, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), u, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), u, err)) return 0, err } @@ -95,7 +97,9 @@ func (s *Spyse) subdomainQueryAPI(domain string, page int) (int, error) { } if err := json.Unmarshal([]byte(response), &results); err != nil { - s.Config().Log.Printf("%s: Failed to unmarshal JSON: %v", s.String(), err) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to unmarshal JSON: %v", s.String(), err), + ) return 0, err } @@ -128,7 +132,7 @@ func (s *Spyse) executeSubdomainQuery(domain string) { url := s.getURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), url, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err)) return } @@ -148,7 +152,7 @@ func (s *Spyse) certQueryAPI(domain string) error { u := s.getCertAPIURL(domain) response, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), u, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), u, err)) return err } @@ -159,7 +163,9 @@ func (s *Spyse) certQueryAPI(domain string) error { } if err := json.Unmarshal([]byte(response), &results); err != nil { - s.Config().Log.Printf("%s: Failed to unmarshal JSON: %v", s.String(), err) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: Failed to unmarshal JSON: %v", s.String(), err), + ) return err } diff --git a/services/sources/spyse_test.go b/services/sources/spyse_test.go index 9786d2f98..7361b933d 100644 --- a/services/sources/spyse_test.go +++ b/services/sources/spyse_test.go @@ -3,7 +3,6 @@ package sources import ( "testing" - "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" ) @@ -14,11 +13,8 @@ func TestSpyse(t *testing.T) { } cfg := setupConfig(domainTest) - - API := new(config.APIKey) - API = cfg.GetAPIKey("spyse") - - if API == nil || API.Key == "" || API.Secret == "" { + api := cfg.GetAPIKey("spyse") + if api == nil || api.Key == "" || api.Secret == "" { t.Errorf("API key data was not provided") return } diff --git a/services/sources/sublist3r.go b/services/sources/sublist3r.go index 46665ad16..e731fe6c4 100644 --- a/services/sources/sublist3r.go +++ b/services/sources/sublist3r.go @@ -5,6 +5,7 @@ package sources import ( "encoding/json" + "fmt" "time" "github.com/OWASP/Amass/config" @@ -19,7 +20,6 @@ import ( type Sublist3rAPI struct { services.BaseService - API *config.APIKey SourceType string RateLimit time.Duration } @@ -71,17 +71,19 @@ func (s *Sublist3rAPI) executeQuery(domain string) { url := s.restURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), url, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err)) return } // Extract the subdomain names from the REST API results var subs []string if err := json.Unmarshal([]byte(page), &subs); err != nil { - s.Config().Log.Printf("%s: %s: %v", s.String(), url, err) + s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err)) return } else if len(subs) == 0 { - s.Config().Log.Printf("%s: %s: The request returned zero results", s.String(), url) + s.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: The request returned zero results", s.String(), url), + ) return } diff --git a/services/sources/teamcymru.go b/services/sources/teamcymru.go index f432df1c8..23723fe8b 100644 --- a/services/sources/teamcymru.go +++ b/services/sources/teamcymru.go @@ -4,6 +4,7 @@ package sources import ( + "fmt" "net" "strconv" "strings" @@ -14,6 +15,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -96,25 +98,33 @@ func (t *TeamCymru) origin(addr string) *requests.ASNRequest { } else if utils.IsIPv6(ip) { name = utils.IPv6NibbleFormat(utils.HexString(ip)) + ".origin6.asn.cymru.com" } else { - t.Config().Log.Printf("%s: %s: Failed to parse the IP address", t.String(), addr) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the IP address", t.String(), addr), + ) return nil } answers, err = t.Pool().Resolve(name, "TXT", resolvers.PriorityHigh) if err != nil { - t.Config().Log.Printf("%s: %s: DNS TXT record query error: %v", t.String(), name, err) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: DNS TXT record query error: %v", t.String(), name, err), + ) return nil } fields := strings.Split(answers[0].Data, " | ") if len(fields) < 5 { - t.Config().Log.Printf("%s: %s: Failed to parse the origin response", t.String(), name) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response", t.String(), name), + ) return nil } asn, err := strconv.Atoi(strings.TrimSpace(fields[0])) if err != nil { - t.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err), + ) return nil } @@ -129,7 +139,7 @@ func (t *TeamCymru) origin(addr string) *requests.ASNRequest { CC: strings.TrimSpace(fields[2]), Registry: strings.TrimSpace(fields[3]), AllocationDate: at, - Netblocks: []string{strings.TrimSpace(fields[1])}, + Netblocks: stringset.New(strings.TrimSpace(fields[1])), Tag: t.SourceType, Source: t.String(), } @@ -142,19 +152,25 @@ func (t *TeamCymru) asnLookup(asn int) *requests.ASNRequest { answers, err = t.Pool().Resolve(name, "TXT", resolvers.PriorityHigh) if err != nil { - t.Config().Log.Printf("%s: %s: DNS TXT record query error: %v", t.String(), name, err) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: DNS TXT record query error: %v", t.String(), name, err), + ) return nil } fields := strings.Split(answers[0].Data, " | ") if len(fields) < 5 { - t.Config().Log.Printf("%s: %s: Failed to parse the origin response", t.String(), name) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response", t.String(), name), + ) return nil } pASN, err := strconv.Atoi(strings.TrimSpace(fields[0])) if err != nil || asn != pASN { - t.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err), + ) return nil } diff --git a/services/sources/threatcrowd.go b/services/sources/threatcrowd.go index a7c268de0..2b932aba7 100644 --- a/services/sources/threatcrowd.go +++ b/services/sources/threatcrowd.go @@ -78,7 +78,7 @@ func (t *ThreatCrowd) executeQuery(domain string) { headers := map[string]string{"Content-Type": "application/json"} page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - t.Config().Log.Printf("%s: %s: %v", t.String(), url, err) + t.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", t.String(), url, err)) return } @@ -95,7 +95,9 @@ func (t *ThreatCrowd) executeQuery(domain string) { } if m.ResponseCode != "1" { - t.Config().Log.Printf("%s: %s: Response code %s", t.String(), url, m.ResponseCode) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Response code %s", t.String(), url, m.ResponseCode), + ) return } diff --git a/services/sources/twitter.go b/services/sources/twitter.go index f95ae6f74..5ffdf9f41 100644 --- a/services/sources/twitter.go +++ b/services/sources/twitter.go @@ -46,7 +46,9 @@ func (t *Twitter) OnStart() error { t.API = t.Config().GetAPIKey(t.String()) if t.API == nil || t.API.Key == "" || t.API.Secret == "" { - t.Config().Log.Printf("%s: API key data was not provided", t.String()) + t.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: API key data was not provided", t.String()), + ) } if t.API != nil && t.API.Key != "" && t.API.Secret != "" { if bearer, err := t.getBearerToken(); err == nil { @@ -99,7 +101,7 @@ func (t *Twitter) executeQuery(domain string) { t.SetActive() search, _, err := t.client.Search.Tweets(searchParams) if err != nil { - t.Config().Log.Printf("%s: %v", t.String(), err) + t.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", t.String(), err)) return } diff --git a/services/sources/twitter_test.go b/services/sources/twitter_test.go index 3157097df..a8621e92d 100644 --- a/services/sources/twitter_test.go +++ b/services/sources/twitter_test.go @@ -3,7 +3,6 @@ package sources import ( "testing" - "github.com/OWASP/Amass/config" "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" ) @@ -14,11 +13,8 @@ func TestTwitter(t *testing.T) { } cfg := setupConfig(domainTest) - - API := new(config.APIKey) - API = cfg.GetAPIKey("twitter") - - if API == nil || API.Key == "" || API.Secret == "" { + api := cfg.GetAPIKey("twitter") + if api == nil || api.Key == "" || api.Secret == "" { t.Errorf("API key data was not provided") return } diff --git a/services/sources/ukgovarchive.go b/services/sources/ukgovarchive.go index 91a4e4a5b..0ff74cf55 100644 --- a/services/sources/ukgovarchive.go +++ b/services/sources/ukgovarchive.go @@ -4,6 +4,8 @@ package sources import ( + "fmt" + "github.com/OWASP/Amass/config" eb "github.com/OWASP/Amass/eventbus" "github.com/OWASP/Amass/requests" @@ -23,7 +25,7 @@ type UKGovArchive struct { } // NewUKGovArchive returns he object initialized, but not yet started. -func NewUKGovArchive(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *UKGovArchive { +func NewUKGovArchive(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *UKGovArchive { u := &UKGovArchive{ domain: "webarchive.nationalarchives.gov.uk", baseURL: "http://webarchive.nationalarchives.gov.uk", @@ -31,7 +33,7 @@ func NewUKGovArchive(c *config.Config, bus *eb.EventBus, pool *resolvers.Resolve filter: utils.NewStringFilter(), } - u.BaseService = *services.NewBaseService(u, "UKGovArchive", c, bus, pool) + u.BaseService = *services.NewBaseService(u, "UKGovArchive", cfg, bus, pool) return u } @@ -67,7 +69,7 @@ func (u *UKGovArchive) executeQuery(sn, domain string) { names, err := crawl(u, u.baseURL, u.domain, sn, domain) if err != nil { - u.Config().Log.Printf("%s: %v", u.String(), err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", u.String(), err)) return } diff --git a/services/sources/umbrella.go b/services/sources/umbrella.go index d59ce929c..c33ea50df 100644 --- a/services/sources/umbrella.go +++ b/services/sources/umbrella.go @@ -14,6 +14,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -43,7 +44,9 @@ func (u *Umbrella) OnStart() error { u.API = u.Config().GetAPIKey(u.String()) if u.API == nil || u.API.Key == "" { - u.Config().Log.Printf("%s: API key data was not provided", u.String()) + u.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: API key data was not provided", u.String()), + ) } go u.processRequests() @@ -92,7 +95,7 @@ func (u *Umbrella) executeDNSQuery(domain string) { url := u.patternSearchRestURL(domain) page, err := utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), url, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err)) return } @@ -108,7 +111,7 @@ func (u *Umbrella) executeDNSQuery(domain string) { url = u.occurrencesRestURL(domain) page, err = utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), url, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err)) return } @@ -128,7 +131,7 @@ func (u *Umbrella) executeDNSQuery(domain string) { url = u.relatedRestURL(domain) page, err = utils.RequestWebPage(url, nil, headers, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), url, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err)) return } @@ -171,24 +174,24 @@ type rWhoisResponse struct { } func (u *Umbrella) collateEmails(record *whoisRecord) []string { - var emails []string + emails := stringset.New() if u.validateScope(record.AdminContactEmail) { - emails = utils.UniqueAppend(emails, record.AdminContactEmail) + emails.InsertMany(record.AdminContactEmail) } if u.validateScope(record.BillingContactEmail) { - emails = utils.UniqueAppend(emails, record.BillingContactEmail) + emails.InsertMany(record.BillingContactEmail) } if u.validateScope(record.RegistrantEmail) { - emails = utils.UniqueAppend(emails, record.RegistrantEmail) + emails.InsertMany(record.RegistrantEmail) } if u.validateScope(record.TechContactEmail) { - emails = utils.UniqueAppend(emails, record.TechContactEmail) + emails.InsertMany(record.TechContactEmail) } if u.validateScope(record.ZoneContactEmail) { - emails = utils.UniqueAppend(emails, record.ZoneContactEmail) + emails.InsertMany(record.ZoneContactEmail) } - return emails + return emails.Slice() } func (u *Umbrella) queryWhois(domain string) *whoisRecord { @@ -199,13 +202,13 @@ func (u *Umbrella) queryWhois(domain string) *whoisRecord { u.SetActive() record, err := utils.RequestWebPage(whoisURL, nil, headers, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), whoisURL, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), whoisURL, err)) return nil } err = json.Unmarshal([]byte(record), &whois) if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), whoisURL, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), whoisURL, err)) return nil } @@ -215,7 +218,7 @@ func (u *Umbrella) queryWhois(domain string) *whoisRecord { } func (u *Umbrella) queryReverseWhois(apiURL string) []string { - var domains []string + domains := stringset.New() headers := u.restHeaders() var whois map[string]rWhoisResponse @@ -225,8 +228,8 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string { fullAPIURL := fmt.Sprintf("%s&offset=%d", apiURL, count) record, err := utils.RequestWebPage(fullAPIURL, nil, headers, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), apiURL, err) - return domains + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), apiURL, err)) + return domains.Slice() } err = json.Unmarshal([]byte(record), &whois) @@ -235,7 +238,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string { if result.TotalResults > 0 { for _, domain := range result.Domains { if domain.Current { - domains = utils.UniqueAppend(domains, domain.Domain) + domains.Insert(domain.Domain) } } } @@ -247,7 +250,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string { u.SetActive() time.Sleep(u.RateLimit) } - return domains + return domains.Slice() } func (u *Umbrella) validateScope(input string) bool { @@ -267,13 +270,13 @@ func (u *Umbrella) executeWhoisQuery(domain string) { return } - var domains []string + domains := stringset.New() emails := u.collateEmails(whoisRecord) if len(emails) > 0 { emailURL := u.reverseWhoisByEmailURL(emails...) for _, d := range u.queryReverseWhois(emailURL) { if !u.Config().IsDomainInScope(d) { - domains = utils.UniqueAppend(domains, d) + domains.Insert(d) } } } @@ -288,7 +291,7 @@ func (u *Umbrella) executeWhoisQuery(domain string) { nsURL := u.reverseWhoisByNSURL(nameservers...) for _, d := range u.queryReverseWhois(nsURL) { if !u.Config().IsDomainInScope(d) { - domains = utils.UniqueAppend(domains, d) + domains.Insert(d) } } } @@ -296,7 +299,7 @@ func (u *Umbrella) executeWhoisQuery(domain string) { if len(domains) > 0 { u.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{ Domain: domain, - NewDomains: domains, + NewDomains: domains.Slice(), Tag: u.SourceType, Source: u.String(), }) diff --git a/services/sources/urlscan.go b/services/sources/urlscan.go index 9753e3cee..418c8a12f 100644 --- a/services/sources/urlscan.go +++ b/services/sources/urlscan.go @@ -14,6 +14,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -43,7 +44,9 @@ func (u *URLScan) OnStart() error { u.API = u.Config().GetAPIKey(u.String()) if u.API == nil || u.API.Key == "" { - u.Config().Log.Printf("%s: API key data was not provided", u.String()) + u.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: API key data was not provided", u.String()), + ) } go u.processRequests() @@ -83,7 +86,7 @@ func (u *URLScan) executeQuery(domain string) { url := u.searchURL(domain) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), url, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err)) return } // Extract the subdomain names from the REST API results @@ -108,12 +111,12 @@ func (u *URLScan) executeQuery(domain string) { } } - var subs []string + subs := stringset.New() for _, id := range ids { - subs = utils.UniqueAppend(subs, u.getSubsFromResult(id)...) + subs.Union(u.getSubsFromResult(id)) } - for _, name := range subs { + for name := range subs { if re.MatchString(name) { u.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{ Name: name, @@ -125,13 +128,13 @@ func (u *URLScan) executeQuery(domain string) { } } -func (u *URLScan) getSubsFromResult(id string) []string { - var subs []string +func (u *URLScan) getSubsFromResult(id string) stringset.Set { + subs := stringset.New() url := u.resultURL(id) page, err := utils.RequestWebPage(url, nil, nil, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), url, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err)) return subs } // Extract the subdomain names from the REST API results @@ -142,7 +145,7 @@ func (u *URLScan) getSubsFromResult(id string) []string { } `json:"lists"` } if err := json.Unmarshal([]byte(page), &data); err == nil { - subs = utils.UniqueAppend(subs, data.Lists.Subdomains...) + subs.InsertMany(data.Lists.Subdomains...) } return subs } @@ -160,7 +163,7 @@ func (u *URLScan) attemptSubmission(domain string) string { body := strings.NewReader(u.submitBody(domain)) page, err := utils.RequestWebPage(url, body, headers, "", "") if err != nil { - u.Config().Log.Printf("%s: %s: %v", u.String(), url, err) + u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err)) return "" } // Extract the subdomain names from the REST API results diff --git a/services/sources/viewdns.go b/services/sources/viewdns.go index 77bd2da48..5b5f3eb79 100644 --- a/services/sources/viewdns.go +++ b/services/sources/viewdns.go @@ -14,6 +14,7 @@ import ( "github.com/OWASP/Amass/requests" "github.com/OWASP/Amass/resolvers" "github.com/OWASP/Amass/services" + "github.com/OWASP/Amass/stringset" "github.com/OWASP/Amass/utils" ) @@ -79,11 +80,11 @@ func (v *ViewDNS) processRequests() { func (v *ViewDNS) executeDNSQuery(domain string) { var unique []string - u := "http://viewdns.info/iphistory/?domain=" + domain + u := v.getIPHistoryURL(domain) // The ViewDNS IP History lookup sometimes reveals interesting results page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - v.Config().Log.Printf("%s: %s: %v", v.String(), u, err) + v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), u, err)) return } @@ -104,46 +105,42 @@ func (v *ViewDNS) executeDNSQuery(domain string) { } func (v *ViewDNS) executeWhoisQuery(domain string) { - u := v.getURL(domain) + u := v.getReverseWhoisURL(domain) page, err := utils.RequestWebPage(u, nil, nil, "", "") if err != nil { - v.Config().Log.Printf("%s: %s: %v", v.String(), u, err) + v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), u, err)) return } // Pull the table we need from the page content table := getViewDNSTable(page) if table == "" { - v.Config().Log.Printf("%s: %s: Failed to discover the table of results", v.String(), u) + v.Bus().Publish(requests.LogTopic, + fmt.Sprintf("%s: %s: Failed to discover the table of results", v.String(), u), + ) return } // Get the list of domain names discovered through the reverse DNS service re := regexp.MustCompile("([a-zA-Z0-9]{1}[a-zA-Z0-9-]{0,61}[a-zA-Z0-9]{1}[.]{1}[a-zA-Z0-9-]+)") subs := re.FindAllStringSubmatch(table, -1) - var matches []string + matches := stringset.New() for _, match := range subs { sub := match[1] if sub != "" { - matches = utils.UniqueAppend(matches, strings.TrimSpace(sub)) + matches.Insert(strings.TrimSpace(sub)) } } if len(matches) > 0 { v.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{ Domain: domain, - NewDomains: matches, + NewDomains: matches.Slice(), Tag: v.SourceType, Source: v.String(), }) } } -func (v *ViewDNS) getURL(domain string) string { - format := "http://viewdns.info/reversewhois/?q=%s" - - return fmt.Sprintf(format, domain) -} - func getViewDNSTable(page string) string { var begin, end int @@ -166,3 +163,13 @@ func getViewDNSTable(page string) string { i = strings.Index(page[begin+i+6:end], "