From fa7a6e2f196a1e3590dc50150915c4547a817d61 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 25 Jul 2019 19:38:16 -0400
Subject: [PATCH 01/33] small formatting edits

---
 cmd/amass/enum.go   | 2 +-
 enum/enum.go        | 2 +-
 examples/minimal.go | 6 +++---
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go
index f2275dd87..92ea30e63 100644
--- a/cmd/amass/enum.go
+++ b/cmd/amass/enum.go
@@ -202,7 +202,7 @@ func runEnumCommand(clArgs []string) {
 
 	rLog, wLog := io.Pipe()
 	e.Config.Log = log.New(wLog, "", log.Lmicroseconds)
-	
+
 	// Check if a configuration file was provided, and if so, load the settings
 	if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, e.Config); err == nil {
 		// Check if a config file was provided that has DNS resolvers specified
diff --git a/enum/enum.go b/enum/enum.go
index befadd4f3..d0f7a6675 100644
--- a/enum/enum.go
+++ b/enum/enum.go
@@ -84,7 +84,7 @@ func NewEnumeration() *Enumeration {
 	if e.Pool == nil {
 		return nil
 	}
-	
+
 	e.dataSources = sources.GetAllSources(e.Config, e.Bus, e.Pool)
 	return e
 }
diff --git a/examples/minimal.go b/examples/minimal.go
index 46c3d74a0..eabef910c 100644
--- a/examples/minimal.go
+++ b/examples/minimal.go
@@ -14,15 +14,15 @@ func main() {
 
 	e := enum.NewEnumeration()
 	if e == nil {
-        return
+		return
 	}
-	
+
 	go func() {
 		for result := range e.Output {
 			fmt.Println(result.Name)
 		}
 	}()
-	
+
 	// Setup the most basic amass configuration
 	e.Config.AddDomain("example.com")
 	e.Start()

From bbdf81a829b83dc8d599b15040f904fdc3f16d5e Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 25 Jul 2019 19:40:13 -0400
Subject: [PATCH 02/33] enhanced the use of the CommonCrawlAPI

---
 services/sources/commoncrawl.go | 131 ++++++++++++++++++++++++++------
 1 file changed, 107 insertions(+), 24 deletions(-)

diff --git a/services/sources/commoncrawl.go b/services/sources/commoncrawl.go
index cc341b0aa..4939b0f00 100644
--- a/services/sources/commoncrawl.go
+++ b/services/sources/commoncrawl.go
@@ -4,7 +4,10 @@
 package sources
 
 import (
+	"bufio"
+	"encoding/json"
 	"net/url"
+	"strings"
 	"time"
 
 	"github.com/OWASP/Amass/config"
@@ -17,18 +20,65 @@ import (
 
 var (
 	commonCrawlIndexes = []string{
-		"CC-MAIN-2019-04",
-		"CC-MAIN-2018-47",
-		"CC-MAIN-2018-39",
-		"CC-MAIN-2018-17",
-		"CC-MAIN-2018-05",
-		"CC-MAIN-2017-43",
-		"CC-MAIN-2017-26",
-		"CC-MAIN-2017-17",
-		"CC-MAIN-2017-04",
-		"CC-MAIN-2016-44",
-		"CC-MAIN-2016-26",
+		"CC-MAIN-2013-20",
+		"CC-MAIN-2013-48",
+		"CC-MAIN-2014-10",
+		"CC-MAIN-2014-15",
+		"CC-MAIN-2014-23",
+		"CC-MAIN-2014-35",
+		"CC-MAIN-2014-41",
+		"CC-MAIN-2014-42",
+		"CC-MAIN-2014-49",
+		"CC-MAIN-2014-52",
+		"CC-MAIN-2015-06",
+		"CC-MAIN-2015-11",
+		"CC-MAIN-2015-14",
+		"CC-MAIN-2015-18",
+		"CC-MAIN-2015-22",
+		"CC-MAIN-2015-27",
+		"CC-MAIN-2015-32",
+		"CC-MAIN-2015-35",
+		"CC-MAIN-2015-40",
+		"CC-MAIN-2015-48",
+		"CC-MAIN-2016-07",
 		"CC-MAIN-2016-18",
+		"CC-MAIN-2016-22",
+		"CC-MAIN-2016-26",
+		"CC-MAIN-2016-30",
+		"CC-MAIN-2016-36",
+		"CC-MAIN-2016-40",
+		"CC-MAIN-2016-44",
+		"CC-MAIN-2016-50",
+		"CC-MAIN-2017-04",
+		"CC-MAIN-2017-09",
+		"CC-MAIN-2017-13",
+		"CC-MAIN-2017-17",
+		"CC-MAIN-2017-22",
+		"CC-MAIN-2017-26",
+		"CC-MAIN-2017-30",
+		"CC-MAIN-2017-34",
+		"CC-MAIN-2017-39",
+		"CC-MAIN-2017-43",
+		"CC-MAIN-2017-47",
+		"CC-MAIN-2017-51",
+		"CC-MAIN-2018-05",
+		"CC-MAIN-2018-09",
+		"CC-MAIN-2018-13",
+		"CC-MAIN-2018-17",
+		"CC-MAIN-2018-22",
+		"CC-MAIN-2018-26",
+		"CC-MAIN-2018-30",
+		"CC-MAIN-2018-34",
+		"CC-MAIN-2018-39",
+		"CC-MAIN-2018-43",
+		"CC-MAIN-2018-47",
+		"CC-MAIN-2018-51",
+		"CC-MAIN-2019-04",
+		"CC-MAIN-2019-09",
+		"CC-MAIN-2019-13",
+		"CC-MAIN-2019-18",
+		"CC-MAIN-2019-22",
+		"CC-MAIN-2019-26",
 	}
 )
 
@@ -44,7 +94,7 @@ type CommonCrawl struct {
 func NewCommonCrawl(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *CommonCrawl {
 	c := &CommonCrawl{
 		baseURL:    "http://index.commoncrawl.org/",
-		SourceType: requests.SCRAPE,
+		SourceType: requests.API,
 	}
 
 	c.BaseService = *services.NewBaseService(c, "CommonCrawl", cfg, bus, pool)
@@ -76,12 +126,13 @@ func (c *CommonCrawl) processRequests() {
 }
 
 func (c *CommonCrawl) executeQuery(domain string) {
+	filter := utils.NewStringFilter()
 	re := c.Config().DomainRegex(domain)
 	if re == nil {
 		return
 	}
 
-	t := time.NewTicker(time.Second)
+	t := time.NewTicker(500 * time.Millisecond)
 	defer t.Stop()
 
 	for _, index := range commonCrawlIndexes {
@@ -91,31 +142,63 @@ func (c *CommonCrawl) executeQuery(domain string) {
 		case <-c.Quit():
 			return
 		case <-t.C:
-			u := c.getURL(index, domain)
+			u := c.getURL(domain, index)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
 				c.Config().Log.Printf("%s: %s: %v", c.String(), u, err)
 				continue
 			}
 
-			for _, sd := range re.FindAllString(page, -1) {
-				c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
-					Name:   cleanName(sd),
-					Domain: domain,
-					Tag:    c.SourceType,
-					Source: c.String(),
-				})
+			for _, url := range c.parseJSON(page) {
+				if name := re.FindString(url); name != "" && !filter.Duplicate(name) {
+					c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
+						Name:   name,
+						Domain: domain,
+						Tag:    c.SourceType,
+						Source: c.String(),
+					})
+				}
 			}
 		}
 	}
 }
 
-func (c *CommonCrawl) getURL(index, domain string) string {
+func (c *CommonCrawl) parseJSON(page string) []string {
+	var urls []string
+	filter := utils.NewStringFilter()
+
+	scanner := bufio.NewScanner(strings.NewReader(page))
+	for scanner.Scan() {
+		// Get the next line of JSON
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+
+		var m struct {
+			URL string `json:"url"`
+		}
+		err := json.Unmarshal([]byte(line), &m)
+		if err != nil {
+			continue
+		}
+
+		if !filter.Duplicate(m.URL) {
+			urls = append(urls, m.URL)
+		}
+	}
+	return urls
+}
+
+func (c *CommonCrawl) getURL(domain, index string) string {
 	u, _ := url.Parse(c.baseURL + index + "-index")
 
 	u.RawQuery = url.Values{
-		"url":    {"*." + domain},
-		"output": {"json"},
+		"url":      {"*." + domain},
+		"output":   {"json"},
+		"filter":   {"=status:200"},
+		"fl":       {"url,status"},
+		"pageSize": {"2000"},
 	}.Encode()
 	return u.String()
 }

From 95944d6d5ac584f47caca637151eb12bc4a9f3bd Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 25 Jul 2019 19:42:59 -0400
Subject: [PATCH 03/33] made performance enhancements and changes to the naming

---
 cmd/amass/intel.go |  10 +--
 intel/intel.go     | 175 +++++++++++++++++++++++----------------------
 2 files changed, 95 insertions(+), 90 deletions(-)

diff --git a/cmd/amass/intel.go b/cmd/amass/intel.go
index efa69a765..73f46edb9 100644
--- a/cmd/amass/intel.go
+++ b/cmd/amass/intel.go
@@ -159,7 +159,7 @@ func runIntelCommand(clArgs []string) {
 		os.Exit(1)
 	}
 
-	ic := intel.NewIntelCollection()
+	ic := intel.NewCollection()
 	if ic == nil {
 		r.Fprintf(color.Error, "%s\n", "No DNS resolvers passed the sanity check")
 		os.Exit(1)
@@ -167,7 +167,7 @@ func runIntelCommand(clArgs []string) {
 
 	rLog, wLog := io.Pipe()
 	ic.Config.Log = log.New(wLog, "", log.Lmicroseconds)
-	
+
 	// Check if a configuration file was provided, and if so, load the settings
 	if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, ic.Config); err == nil {
 		// Check if a config file was provided that has DNS resolvers specified
@@ -210,7 +210,7 @@ func runIntelCommand(clArgs []string) {
 	processIntelOutput(ic, &args, rLog)
 }
 
-func processIntelOutput(ic *intel.IntelCollection, args *intelArgs, pipe *io.PipeReader) {
+func processIntelOutput(ic *intel.Collection, args *intelArgs, pipe *io.PipeReader) {
 	var err error
 
 	// Prepare output file paths
@@ -272,7 +272,7 @@ func processIntelOutput(ic *intel.IntelCollection, args *intelArgs, pipe *io.Pip
 }
 
 // If the user interrupts the program, print the summary information
-func intelSignalHandler(ic *intel.IntelCollection) {
+func intelSignalHandler(ic *intel.Collection) {
 	quit := make(chan os.Signal, 1)
 
 	signal.Notify(quit, os.Interrupt, syscall.SIGTERM)
@@ -352,7 +352,7 @@ func processIntelInputFiles(args *intelArgs) error {
 }
 
 // Setup the amass intelligence collection settings
-func updateIntelConfiguration(ic *intel.IntelCollection, args *intelArgs) error {
+func updateIntelConfiguration(ic *intel.Collection, args *intelArgs) error {
 	if args.Options.Active {
 		ic.Config.Active = true
 	}
diff --git a/intel/intel.go b/intel/intel.go
index ab3b12ece..8e007d8c7 100644
--- a/intel/intel.go
+++ b/intel/intel.go
@@ -23,8 +23,8 @@ import (
 	"github.com/OWASP/Amass/utils"
 )
 
-// IntelCollection is the object type used to execute a open source information gathering with Amass.
-type IntelCollection struct {
+// Collection is the object type used to execute a open source information gathering with Amass.
+type Collection struct {
 	Config *config.Config
 	Bus    *eb.EventBus
 	Pool   *resolvers.ResolverPool
@@ -44,9 +44,9 @@ type IntelCollection struct {
 	activeChan chan struct{}
 }
 
-// NewIntelCollection returns an initialized IntelCollection object that has not been started yet.
-func NewIntelCollection() *IntelCollection {
-	ic := &IntelCollection{
+// NewCollection returns an initialized Collection object that has not been started yet.
+func NewCollection() *Collection {
+	c := &Collection{
 		Config:     &config.Config{Log: log.New(ioutil.Discard, "", 0)},
 		Bus:        eb.NewEventBus(),
 		Pool:       resolvers.NewResolverPool(nil),
@@ -57,28 +57,28 @@ func NewIntelCollection() *IntelCollection {
 		domainChan: make(chan *requests.Output, 100),
 		activeChan: make(chan struct{}, 100),
 	}
-	if ic.Pool == nil {
+	if c.Pool == nil {
 		return nil
 	}
-	return ic
+	return c
 }
 
 // HostedDomains uses open source intelligence to discover root domain names in the target infrastructure.
-func (ic *IntelCollection) HostedDomains() error {
-	if ic.Output == nil {
+func (c *Collection) HostedDomains() error {
+	if c.Output == nil {
 		return errors.New("The intelligence collection did not have an output channel")
-	} else if err := ic.Config.CheckSettings(); err != nil {
+	} else if err := c.Config.CheckSettings(); err != nil {
 		return err
 	}
 
-	go ic.startAddressRanges()
-	go ic.processCIDRs()
+	go c.startAddressRanges()
+	go c.processCIDRs()
 	go func() {
-		for _, cidr := range ic.Config.CIDRs {
-			ic.cidrChan <- cidr
+		for _, cidr := range c.Config.CIDRs {
+			c.cidrChan <- cidr
 		}
 	}()
-	ic.asnsToCIDRs()
+	c.asnsToCIDRs()
 
 	var active bool
 	filter := utils.NewStringFilter()
@@ -86,50 +86,55 @@ func (ic *IntelCollection) HostedDomains() error {
 loop:
 	for {
 		select {
-		case <-ic.Done:
+		case <-c.Done:
 			break loop
 		case <-t.C:
 			if !active {
-				close(ic.Done)
+				close(c.Done)
 			}
 			active = false
-		case <-ic.activeChan:
+		case <-c.activeChan:
 			active = true
-		case d := <-ic.domainChan:
+		case d := <-c.domainChan:
 			active = true
 			if !filter.Duplicate(d.Domain) {
-				ic.Output <- d
+				c.Output <- d
 			}
 		}
 	}
 	t.Stop()
-	close(ic.Output)
+	close(c.Output)
 	return nil
 }
 
-func (ic *IntelCollection) startAddressRanges() {
-	for _, addr := range ic.Config.Addresses {
-		ic.Config.SemMaxDNSQueries.Acquire(1)
-		go ic.investigateAddr(addr.String())
+func (c *Collection) startAddressRanges() {
+	for _, addr := range c.Config.Addresses {
+		c.Config.SemMaxDNSQueries.Acquire(1)
+		go c.investigateAddr(addr.String())
 	}
 }
 
-func (ic *IntelCollection) processCIDRs() {
+func (c *Collection) processCIDRs() {
 	for {
 		select {
-		case <-ic.Done:
+		case <-c.Done:
 			return
-		case cidr := <-ic.cidrChan:
+		case cidr := <-c.cidrChan:
+			// Skip IPv6 netblocks, since they are simply too large
+			if ip := cidr.IP.Mask(cidr.Mask); utils.IsIPv6(ip) {
+				continue
+			}
+
 			for _, addr := range utils.NetHosts(cidr) {
-				ic.Config.SemMaxDNSQueries.Acquire(1)
-				go ic.investigateAddr(addr.String())
+				c.Config.SemMaxDNSQueries.Acquire(1)
+				go c.investigateAddr(addr.String())
 			}
 		}
 	}
 }
 
-func (ic *IntelCollection) investigateAddr(addr string) {
-	defer ic.Config.SemMaxDNSQueries.Release(1)
+func (c *Collection) investigateAddr(addr string) {
+	defer c.Config.SemMaxDNSQueries.Release(1)
 
 	ip := net.ParseIP(addr)
 	if ip == nil {
@@ -137,10 +142,10 @@ func (ic *IntelCollection) investigateAddr(addr string) {
 	}
 
 	addrinfo := requests.AddressInfo{Address: ip}
-	ic.activeChan <- struct{}{}
-	if _, answer, err := ic.Pool.ReverseDNS(addr); err == nil {
-		if d := strings.TrimSpace(ic.Pool.SubdomainToDomain(answer)); d != "" {
-			ic.domainChan <- &requests.Output{
+	c.activeChan <- struct{}{}
+	if _, answer, err := c.Pool.ReverseDNS(addr); err == nil {
+		if d := strings.TrimSpace(c.Pool.SubdomainToDomain(answer)); d != "" {
+			c.domainChan <- &requests.Output{
 				Name:      d,
 				Domain:    d,
 				Addresses: []requests.AddressInfo{addrinfo},
@@ -150,37 +155,37 @@ func (ic *IntelCollection) investigateAddr(addr string) {
 		}
 	}
 
-	ic.activeChan <- struct{}{}
-	if !ic.Config.Active {
+	c.activeChan <- struct{}{}
+	if !c.Config.Active {
 		return
 	}
 
-	for _, name := range utils.PullCertificateNames(addr, ic.Config.Ports) {
+	for _, name := range utils.PullCertificateNames(addr, c.Config.Ports) {
 		if n := strings.TrimSpace(name); n != "" {
-			ic.domainChan <- &requests.Output{
+			c.domainChan <- &requests.Output{
 				Name:      n,
-				Domain:    ic.Pool.SubdomainToDomain(n),
+				Domain:    c.Pool.SubdomainToDomain(n),
 				Addresses: []requests.AddressInfo{addrinfo},
 				Tag:       requests.CERT,
 				Source:    "Active Cert",
 			}
 		}
 	}
-	ic.activeChan <- struct{}{}
+	c.activeChan <- struct{}{}
 }
 
-func (ic *IntelCollection) asnsToCIDRs() {
-	if len(ic.Config.ASNs) == 0 {
+func (c *Collection) asnsToCIDRs() {
+	if len(c.Config.ASNs) == 0 {
 		return
 	}
 
-	ic.Bus.Subscribe(requests.NewASNTopic, ic.updateNetCache)
-	defer ic.Bus.Unsubscribe(requests.NewASNTopic, ic.updateNetCache)
+	c.Bus.Subscribe(requests.NewASNTopic, c.updateNetCache)
+	defer c.Bus.Unsubscribe(requests.NewASNTopic, c.updateNetCache)
 
-	srcs := sources.GetAllSources(ic.Config, ic.Bus, ic.Pool)
+	srcs := sources.GetAllSources(c.Config, c.Bus, c.Pool)
 	// Select the data sources desired by the user
-	if len(ic.Config.DisabledDataSources) > 0 {
-		srcs = ExcludeDisabledDataSources(srcs, ic.Config)
+	if len(c.Config.DisabledDataSources) > 0 {
+		srcs = ExcludeDisabledDataSources(srcs, c.Config)
 	}
 	// Keep only the data sources that successfully start
 	var keep []services.Service
@@ -195,7 +200,7 @@ func (ic *IntelCollection) asnsToCIDRs() {
 	srcs = keep
 
 	// Send the ASN requests to the data sources
-	for _, asn := range ic.Config.ASNs {
+	for _, asn := range c.Config.ASNs {
 		for _, src := range srcs {
 			src.SendASNRequest(&requests.ASNRequest{ASN: asn})
 		}
@@ -203,10 +208,10 @@ func (ic *IntelCollection) asnsToCIDRs() {
 
 	t := time.NewTicker(5 * time.Second)
 	defer t.Stop()
-	defer ic.sendNetblockCIDRs()
+	defer c.sendNetblockCIDRs()
 	for {
 		select {
-		case <-ic.Done:
+		case <-c.Done:
 			return
 		case <-t.C:
 			done := true
@@ -223,49 +228,49 @@ func (ic *IntelCollection) asnsToCIDRs() {
 	}
 }
 
-func (ic *IntelCollection) sendNetblockCIDRs() {
-	ic.netLock.Lock()
-	defer ic.netLock.Unlock()
+func (c *Collection) sendNetblockCIDRs() {
+	c.netLock.Lock()
+	defer c.netLock.Unlock()
 
 	filter := utils.NewStringFilter()
-	for _, record := range ic.netCache {
+	for _, record := range c.netCache {
 		for _, netblock := range record.Netblocks {
 			_, ipnet, err := net.ParseCIDR(netblock)
 			if err == nil && !filter.Duplicate(ipnet.String()) {
-				ic.cidrChan <- ipnet
+				c.cidrChan <- ipnet
 			}
 		}
 	}
 }
 
-func (ic *IntelCollection) updateNetCache(req *requests.ASNRequest) {
-	ic.netLock.Lock()
-	defer ic.netLock.Unlock()
+func (c *Collection) updateNetCache(req *requests.ASNRequest) {
+	c.netLock.Lock()
+	defer c.netLock.Unlock()
 
-	if _, found := ic.netCache[req.ASN]; !found {
-		ic.netCache[req.ASN] = req
+	if _, found := c.netCache[req.ASN]; !found {
+		c.netCache[req.ASN] = req
 		return
 	}
 
-	c := ic.netCache[req.ASN]
+	entry := c.netCache[req.ASN]
 	// This is additional information for an ASN entry
-	if c.Prefix == "" && req.Prefix != "" {
-		c.Prefix = req.Prefix
+	if entry.Prefix == "" && req.Prefix != "" {
+		entry.Prefix = req.Prefix
 	}
-	if c.CC == "" && req.CC != "" {
-		c.CC = req.CC
+	if entry.CC == "" && req.CC != "" {
+		entry.CC = req.CC
 	}
-	if c.Registry == "" && req.Registry != "" {
-		c.Registry = req.Registry
+	if entry.Registry == "" && req.Registry != "" {
+		entry.Registry = req.Registry
 	}
-	if c.AllocationDate.IsZero() && !req.AllocationDate.IsZero() {
-		c.AllocationDate = req.AllocationDate
+	if entry.AllocationDate.IsZero() && !req.AllocationDate.IsZero() {
+		entry.AllocationDate = req.AllocationDate
 	}
-	if c.Description == "" && req.Description != "" {
-		c.Description = req.Description
+	if entry.Description == "" && req.Description != "" {
+		entry.Description = req.Description
 	}
-	c.Netblocks = utils.UniqueAppend(c.Netblocks, req.Netblocks...)
-	ic.netCache[req.ASN] = c
+	entry.Netblocks = utils.UniqueAppend(entry.Netblocks, req.Netblocks...)
+	c.netCache[req.ASN] = entry
 }
 
 // LookupASNsByName returns requests.ASNRequest objects for autonomous systems with
@@ -302,13 +307,13 @@ func LookupASNsByName(s string) ([]*requests.ASNRequest, error) {
 }
 
 // ReverseWhois returns domain names that are related to the domains provided
-func (ic *IntelCollection) ReverseWhois() error {
+func (c *Collection) ReverseWhois() error {
 	filter := utils.NewStringFilter()
 
 	collect := func(req *requests.WhoisRequest) {
 		for _, d := range req.NewDomains {
 			if !filter.Duplicate(d) {
-				ic.Output <- &requests.Output{
+				c.Output <- &requests.Output{
 					Name:   d,
 					Domain: d,
 					Tag:    req.Tag,
@@ -317,13 +322,13 @@ func (ic *IntelCollection) ReverseWhois() error {
 			}
 		}
 	}
-	ic.Bus.Subscribe(requests.NewWhoisTopic, collect)
-	defer ic.Bus.Unsubscribe(requests.NewWhoisTopic, collect)
+	c.Bus.Subscribe(requests.NewWhoisTopic, collect)
+	defer c.Bus.Unsubscribe(requests.NewWhoisTopic, collect)
 
-	srcs := sources.GetAllSources(ic.Config, ic.Bus, ic.Pool)
+	srcs := sources.GetAllSources(c.Config, c.Bus, c.Pool)
 	// Select the data sources desired by the user
-	if len(ic.Config.DisabledDataSources) > 0 {
-		srcs = ExcludeDisabledDataSources(srcs, ic.Config)
+	if len(c.Config.DisabledDataSources) > 0 {
+		srcs = ExcludeDisabledDataSources(srcs, c.Config)
 	}
 	// Keep only the data sources that successfully start
 	var keep []services.Service
@@ -338,7 +343,7 @@ func (ic *IntelCollection) ReverseWhois() error {
 	srcs = keep
 
 	// Send the whois requests to the data sources
-	for _, domain := range ic.Config.Domains() {
+	for _, domain := range c.Config.Domains() {
 		for _, src := range srcs {
 			src.SendWhoisRequest(&requests.WhoisRequest{Domain: domain})
 		}
@@ -348,7 +353,7 @@ func (ic *IntelCollection) ReverseWhois() error {
 loop:
 	for {
 		select {
-		case <-ic.Done:
+		case <-c.Done:
 			break loop
 		case <-t.C:
 			done := true
@@ -364,7 +369,7 @@ loop:
 		}
 	}
 	t.Stop()
-	close(ic.Output)
+	close(c.Output)
 	return nil
 }
 

From 7137cc41ec8b1e00859077f191b750bff1d7ac06 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 25 Jul 2019 19:43:38 -0400
Subject: [PATCH 04/33] updated the source installation instructions

---
 doc/install.md | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/doc/install.md b/doc/install.md
index aa6e9ac00..afd1db9fd 100644
--- a/doc/install.md
+++ b/doc/install.md
@@ -32,7 +32,15 @@ docker run -v ~/amass:/amass/ amass enum -brute -w /wordlists/all.txt -d example
 
 ## From Source
 
-If you prefer to build your own binary from the latest release of the source code, make sure you have a correctly configured **Go >= 1.10** environment. More information about how to achieve this can be found [on the golang website.](https://golang.org/doc/install) Then, take the following steps:
+If you prefer to build your own binary from the latest release of the source code, make sure you have a correctly configured **Go >= 1.12** environment. More information about how to achieve this can be found [on the golang website.](https://golang.org/doc/install).
+
+If you are not utilizing Go Modules, then you can simply execute the following command:
+
+```bash
+go get -u github.com/OWASP/Amass/...
+```
+
+If you would like to build Amass using Go Modules to ensure the proper dependencies, then perform the following steps:
 
 1. Download OWASP Amass:
 
@@ -48,7 +56,7 @@ Ignore any error messages regarding what was pulled down.
 export GO111MODULE=on
 ```
 
-3. Next, build the binaries from the project source code:
+3. Next, build the binary from the project source code:
 
 ```bash
 cd $GOPATH/src/github.com/OWASP/Amass
@@ -56,9 +64,7 @@ cd $GOPATH/src/github.com/OWASP/Amass
 go install ./...
 ```
 
-At this point, the binaries should be in *$GOPATH/bin*.
-
-4. Several wordlists can be found in the following directory:
+At this point, the binary should be in *$GOPATH/bin*. Several wordlists for performing DNS name alterations and brute forcing can be found in the following directory:
 
 ```bash
 ls $GOPATH/src/github.com/OWASP/Amass/wordlists/
@@ -66,9 +72,9 @@ ls $GOPATH/src/github.com/OWASP/Amass/wordlists/
 
 ## Packages Maintained by the Amass Project
 
-### Homebrew (macOS)
+### Homebrew
 
-For **Homebrew** on **Mac**, the following two commands will install Amass into your macOS environment:
+For **Homebrew**, the following two commands will install Amass into your environment:
 
 ```bash
 brew tap caffix/amass

From 01b9a62b21c9c7ae6fc3f2ede8a962c8902e5630 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 25 Jul 2019 19:44:22 -0400
Subject: [PATCH 05/33] improved the performance of StringFilter

---
 utils/misc.go | 43 +++++++++----------------------------------
 1 file changed, 9 insertions(+), 34 deletions(-)

diff --git a/utils/misc.go b/utils/misc.go
index 39f75f0f5..f8c7d737a 100644
--- a/utils/misc.go
+++ b/utils/misc.go
@@ -54,52 +54,25 @@ func getWordList(reader io.Reader) []string {
 	return words
 }
 
-type filterRequest struct {
-	String string
-	Result chan bool
-}
-
 // StringFilter implements an object that performs filtering of strings
 // to ensure that only unique items get through the filter.
 type StringFilter struct {
-	filter   *cfilter.CFilter
-	requests chan filterRequest
-	quit     chan struct{}
+	filter *cfilter.CFilter
 }
 
 // NewStringFilter returns an initialized StringFilter.
 func NewStringFilter() *StringFilter {
-	sf := &StringFilter{
-		filter:   cfilter.New(),
-		requests: make(chan filterRequest),
-		quit:     make(chan struct{}),
-	}
-	go sf.processRequests()
-	return sf
+	return &StringFilter{filter: cfilter.New()}
 }
 
 // Duplicate checks if the name provided has been seen before by this filter.
 func (sf *StringFilter) Duplicate(s string) bool {
-	result := make(chan bool)
-
-	sf.requests <- filterRequest{String: s, Result: result}
-	return <-result
-}
-
-func (sf *StringFilter) processRequests() {
-	for {
-		select {
-		case <-sf.quit:
-			return
-		case r := <-sf.requests:
-			if sf.filter.Lookup([]byte(r.String)) {
-				r.Result <- true
-			} else {
-				sf.filter.Insert([]byte(r.String))
-				r.Result <- false
-			}
-		}
+	if sf.filter.Lookup([]byte(s)) {
+		return true
 	}
+
+	sf.filter.Insert([]byte(s))
+	return false
 }
 
 // SubdomainRegex returns a Regexp object initialized to match
@@ -178,6 +151,7 @@ func RemoveAsteriskLabel(s string) string {
 	return strings.Join(labels[index:], ".")
 }
 
+// ExpandMask will return a slice of words that a "hashcat-style" mask matches.
 func ExpandMask(word string) ([]string, error) {
 	var expanded []string
 	var chars string
@@ -218,6 +192,7 @@ func ExpandMask(word string) ([]string, error) {
 	return expanded, nil
 }
 
+// ExpandMaskWordlist performs ExpandMask on a slice of words.
 func ExpandMaskWordlist(wordlist []string) ([]string, error) {
 	var newWordlist []string
 	var newWords []string

From 8d3773782eca56562f43cc86c1cbfe2a53e8185d Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 25 Jul 2019 22:51:58 -0400
Subject: [PATCH 06/33] new subdomains are probed with typical hostnames to
 initialize brute forcing

---
 services/namesrv.go | 33 +++++++++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/services/namesrv.go b/services/namesrv.go
index 716047d2d..8ddda967c 100644
--- a/services/namesrv.go
+++ b/services/namesrv.go
@@ -16,6 +16,23 @@ import (
 	"github.com/OWASP/Amass/utils"
 )
 
+var (
+	topNames = []string{
+		"www",
+		"online",
+		"webserver",
+		"ns1",
+		"mail",
+		"smtp",
+		"webmail",
+		"prod",
+		"test",
+		"vpn",
+		"ftp",
+		"ssh",
+	}
+)
+
 type timesRequest struct {
 	Subdomain string
 	Times     chan int
@@ -54,6 +71,7 @@ func (ns *NameService) OnStart() error {
 
 	ns.Bus().Subscribe(requests.NewNameTopic, ns.newNameEvent)
 	ns.Bus().Subscribe(requests.NameResolvedTopic, ns.Resolved)
+	ns.Bus().Subscribe(requests.NewSubdomainTopic, ns.probeSubdomain)
 	go ns.processTimesRequests()
 	go ns.processRequests()
 	return nil
@@ -122,6 +140,21 @@ func (ns *NameService) Resolved(req *requests.DNSRequest) {
 	}
 }
 
+func (ns *NameService) probeSubdomain(req *requests.DNSRequest, times int) {
+	if ns.Config().Passive || times > 1 {
+		return
+	}
+
+	for _, name := range topNames {
+		go ns.newNameEvent(&requests.DNSRequest{
+			Name:   name + "." + req.Name,
+			Domain: req.Domain,
+			Tag:    requests.ALT,
+			Source: ns.String(),
+		})
+	}
+}
+
 func (ns *NameService) checkSubdomain(req *requests.DNSRequest) {
 	labels := strings.Split(req.Name, ".")
 	num := len(labels)

From 7dc902e34ec334b2ba7f8aeb342325d6bd7509a6 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Fri, 26 Jul 2019 00:40:16 -0400
Subject: [PATCH 07/33] improvements to the subdomain probing functionality

---
 services/namesrv.go | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/services/namesrv.go b/services/namesrv.go
index 8ddda967c..e0372ce1a 100644
--- a/services/namesrv.go
+++ b/services/namesrv.go
@@ -71,7 +71,6 @@ func (ns *NameService) OnStart() error {
 
 	ns.Bus().Subscribe(requests.NewNameTopic, ns.newNameEvent)
 	ns.Bus().Subscribe(requests.NameResolvedTopic, ns.Resolved)
-	ns.Bus().Subscribe(requests.NewSubdomainTopic, ns.probeSubdomain)
 	go ns.processTimesRequests()
 	go ns.processRequests()
 	return nil
@@ -137,21 +136,17 @@ func (ns *NameService) Resolved(req *requests.DNSRequest) {
 
 	if ns.Config().IsDomainInScope(req.Name) {
 		ns.checkSubdomain(req)
-	}
-}
-
-func (ns *NameService) probeSubdomain(req *requests.DNSRequest, times int) {
-	if ns.Config().Passive || times > 1 {
-		return
-	}
 
-	for _, name := range topNames {
-		go ns.newNameEvent(&requests.DNSRequest{
-			Name:   name + "." + req.Name,
-			Domain: req.Domain,
-			Tag:    requests.ALT,
-			Source: ns.String(),
-		})
+		if ns.Config().BruteForcing && ns.Config().Recursive {
+			for _, name := range topNames {
+				go ns.newNameEvent(&requests.DNSRequest{
+					Name:   name + "." + req.Name,
+					Domain: req.Domain,
+					Tag:    requests.ALT,
+					Source: ns.String(),
+				})
+			}
+		}
 	}
 }
 

From 29021d86314b2821eb75b6d532a0a89291a9ae35 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Fri, 26 Jul 2019 00:41:24 -0400
Subject: [PATCH 08/33] updated dependency info for #228

---
 go.mod | 14 ++++++++------
 go.sum | 15 +++++++++++++++
 2 files changed, 23 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 9b427c6c8..b8ce7783c 100644
--- a/go.mod
+++ b/go.mod
@@ -11,23 +11,25 @@ require (
 	github.com/caffix/cloudflare-roundtripper v0.0.0-20181218223503-4c29d231c9cb
 	github.com/cayleygraph/cayley v0.7.5
 	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
-	github.com/chromedp/cdproto v0.0.0-20190714225024-2508b04fd5cb // indirect
+	github.com/chromedp/cdproto v0.0.0-20190721111337-61a0348ea0b1 // indirect
 	github.com/chromedp/chromedp v0.3.1 // indirect
 	github.com/dghubble/go-twitter v0.0.0-20190719072343-39e5462e111f
 	github.com/fatih/color v1.7.0
 	github.com/fpfeng/httpcache v0.0.0-20181220163524-ab6bbcc7c729 // indirect
-	github.com/geziyor/geziyor v0.0.0-20190714003752-df37629d4d5b
+	github.com/geziyor/geziyor v0.0.0-20190721090841-762854e5113a
 	github.com/go-ini/ini v1.44.0
 	github.com/go-kit/kit v0.9.0 // indirect
 	github.com/go-sql-driver/mysql v1.4.1 // indirect
 	github.com/gobwas/ws v1.0.2 // indirect
 	github.com/gogo/protobuf v1.2.1 // indirect
 	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70 // indirect
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/irfansharif/cfilter v0.1.1
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/johnnadratowski/golang-neo4j-bolt-driver v0.0.0-20181101021923-6b24c0085aae
+	github.com/json-iterator/go v1.1.7 // indirect
 	github.com/kisielk/errcheck v1.2.0 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/lib/pq v1.2.0
@@ -50,11 +52,11 @@ require (
 	golang.org/x/exp v0.0.0-20190718202018-cfdd5522f6f6 // indirect
 	golang.org/x/image v0.0.0-20190703141733-d6a02ce849c9 // indirect
 	golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 // indirect
-	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
+	golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 // indirect
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
-	golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 // indirect
-	golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386 // indirect
-	google.golang.org/grpc v1.22.0 // indirect
+	golang.org/x/sys v0.0.0-20190726002231-94b544f455ef // indirect
+	golang.org/x/tools v0.0.0-20190725161231-2e34cfcb95cb // indirect
+	google.golang.org/grpc v1.22.1 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/yaml.v2 v2.2.2 // indirect
diff --git a/go.sum b/go.sum
index afd918946..70d3ddc15 100644
--- a/go.sum
+++ b/go.sum
@@ -32,6 +32,8 @@ github.com/chromedp/cdproto v0.0.0-20190609032908-dd39f0bf0a54 h1:2NlKweNkC3yy6I
 github.com/chromedp/cdproto v0.0.0-20190609032908-dd39f0bf0a54/go.mod h1:5NWqr1Ri5aJB5uSvUXfVpbBslleS+eMjspUWv2Lcaow=
 github.com/chromedp/cdproto v0.0.0-20190714225024-2508b04fd5cb h1:NzR98imHjC6LsmogFqMAOhJWqixFwnWcQQTc1qTJafU=
 github.com/chromedp/cdproto v0.0.0-20190714225024-2508b04fd5cb/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
+github.com/chromedp/cdproto v0.0.0-20190721111337-61a0348ea0b1 h1:TFdsyNN0dHau0EDzAnCv56ZgsAlucefia13zYRc18LA=
+github.com/chromedp/cdproto v0.0.0-20190721111337-61a0348ea0b1/go.mod h1:0YChpVzuLJC5CPr+x3xkHN6Z8KOSXjNbL7qV8Wc4GW0=
 github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05 h1:5iy45UjpWvkgTcd7GrGQSPr7sifrp9nNweI/eAsMjGE=
 github.com/chromedp/chromedp v0.3.1-0.20190617065505-d55cf9043e05/go.mod h1:MsTqWB2yT7cErDFnF1F3y0PN8i/a/qQj+0GXKLW/I3s=
 github.com/chromedp/chromedp v0.3.1 h1:QV2NmprlDOMssqCjxemgHehVldLB2wp+Spa+bRbQGmc=
@@ -58,6 +60,8 @@ github.com/geziyor/geziyor v0.0.0-20190703175417-71683ec6de71 h1:DD67izwjI7a9nMV
 github.com/geziyor/geziyor v0.0.0-20190703175417-71683ec6de71/go.mod h1:oL2vq/ZSixF5Wy8Ac+h0MazxgGgCCqvp2tXorUk+fqw=
 github.com/geziyor/geziyor v0.0.0-20190714003752-df37629d4d5b h1:fQWkKQc1pP88SLP2OoUWmPLGiWctkyU9RtzelL8hxbM=
 github.com/geziyor/geziyor v0.0.0-20190714003752-df37629d4d5b/go.mod h1:ByMdw1IoR4O5LTZW4LrkeYrqOb39uG9M6PFo1gjWKNE=
+github.com/geziyor/geziyor v0.0.0-20190721090841-762854e5113a h1:MYb8yIB2bIkeNjHG0YtgvJrh+2IqEHCjymGowc+TYpg=
+github.com/geziyor/geziyor v0.0.0-20190721090841-762854e5113a/go.mod h1:ByMdw1IoR4O5LTZW4LrkeYrqOb39uG9M6PFo1gjWKNE=
 github.com/go-ini/ini v1.42.0 h1:TWr1wGj35+UiWHlBA8er89seFXxzwFn11spilrrj+38=
 github.com/go-ini/ini v1.42.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w=
@@ -101,9 +105,11 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190723021845-34ac40c74b70/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
@@ -120,6 +126,7 @@ github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhB
 github.com/johnnadratowski/golang-neo4j-bolt-driver v0.0.0-20181101021923-6b24c0085aae h1:YovtHFY2RXA6muqKixVlJlLBvBNNjiLTYeCYZTn/QDo=
 github.com/johnnadratowski/golang-neo4j-bolt-driver v0.0.0-20181101021923-6b24c0085aae/go.mod h1:xwUw3ZE1/D9drQgpluhRs4peTMKm1tQEZ4p7DrpyqwE=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -152,7 +159,9 @@ github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI=
 github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -240,6 +249,8 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
 golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk=
+golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
@@ -267,6 +278,8 @@ golang.org/x/sys v0.0.0-20190610081024-1e42afee0f76/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 h1:LepdCS8Gf/MVejFIt8lsiexZATdoGVyp5bcyS+rYoUI=
 golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190726002231-94b544f455ef h1:vwqipsjwy3Y8/PQk/LmiaFjos8aOnU6Tt6oRXKD3org=
+golang.org/x/sys v0.0.0-20190726002231-94b544f455ef/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
@@ -287,6 +300,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
+golang.org/x/tools v0.0.0-20190725161231-2e34cfcb95cb/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@@ -303,6 +317,7 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

From fc7e75ac8e4cd7dbb1e7d3a03491fbd36fe74d10 Mon Sep 17 00:00:00 2001
From: Kian Jamali <kian.jamali@citrix.com>
Date: Wed, 24 Jul 2019 22:50:42 -0700
Subject: [PATCH 09/33] added the shodan_test.go file

---
 services/sources/shodan_test.go | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 services/sources/shodan_test.go

diff --git a/services/sources/shodan_test.go b/services/sources/shodan_test.go
new file mode 100644
index 000000000..e69de29bb

From bd66cb0549cfce1fe431662f78906b956b9e9c8b Mon Sep 17 00:00:00 2001
From: Kian Jamali <kian.jamali@citrix.com>
Date: Thu, 25 Jul 2019 01:25:46 -0700
Subject: [PATCH 10/33] implemented the TestShodan function

---
 services/sources/shodan_test.go | 38 +++++++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/services/sources/shodan_test.go b/services/sources/shodan_test.go
index e69de29bb..f20a2c84a 100644
--- a/services/sources/shodan_test.go
+++ b/services/sources/shodan_test.go
@@ -0,0 +1,38 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/OWASP/Amass/config"
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+)
+
+func TestShodan(t *testing.T) {
+	if *networkTest == false || *configPath == "" {
+		return
+	}
+
+	cfg := setupConfig(domainTest)
+
+	API := new(config.APIKey)
+	API = cfg.GetAPIKey("shodan")
+
+	if API == nil || API.Key == "" {
+		t.Errorf("API key data was not provided")
+		return
+	}
+	
+	bus, out := setupEventBus(requests.NewNameTopic)
+	defer bus.Stop()
+
+	pool := resolvers.NewResolverPool(nil)
+	defer pool.Stop()
+
+	srv := NewShodan(cfg, bus, pool)
+
+	result := testService(srv, out)
+	if result < expectedTest {
+		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
+	}
+}
\ No newline at end of file

From 5eee45a59bb15ca82ec2f27a58319d161df6890c Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Sun, 28 Jul 2019 20:17:31 -0400
Subject: [PATCH 11/33] updated comments for queue implementation

---
 utils/queue.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/utils/queue.go b/utils/queue.go
index 239eccae5..b966f86bb 100644
--- a/utils/queue.go
+++ b/utils/queue.go
@@ -58,7 +58,7 @@ func (q *Queue) Next() (interface{}, bool) {
 	return element.Data, true
 }
 
-// Empty return true if the Queue is empty.
+// Empty returns true if the Queue is empty.
 func (q *Queue) Empty() bool {
 	q.Lock()
 	defer q.Unlock()

From 299c1262f33984c07a91f1f799fc8f6fad1d69b7 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Sun, 28 Jul 2019 20:19:13 -0400
Subject: [PATCH 12/33] changed the default number of concurrent DNS queries to
 2500

---
 config/config.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/config/config.go b/config/config.go
index 8e23a895c..e0e56a594 100644
--- a/config/config.go
+++ b/config/config.go
@@ -29,8 +29,9 @@ const (
 	// DefaultOutputDirectory is the name of the directory used for output files, such as the graph database.
 	DefaultOutputDirectory = "amass"
 
-	defaultWordlistURL    = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/namelist.txt"
-	defaultAltWordlistURL = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/alterations.txt"
+	defaultConcurrentDNSQueries = 2500
+	defaultWordlistURL          = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/namelist.txt"
+	defaultAltWordlistURL       = "https://raw.githubusercontent.com/OWASP/Amass/master/wordlists/alterations.txt"
 )
 
 // Config passes along Amass configuration settings and options.
@@ -145,7 +146,7 @@ func (c *Config) CheckSettings() error {
 		return errors.New("Active enumeration cannot be performed without DNS resolution")
 	}
 	if c.MaxDNSQueries <= 0 {
-		c.MaxDNSQueries = 1000
+		c.MaxDNSQueries = defaultConcurrentDNSQueries
 	}
 	if len(c.Ports) == 0 {
 		c.Ports = []int{443}

From dec4f4cecae4c88fcc9c86bd430fa95307920ee6 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Sun, 28 Jul 2019 20:24:02 -0400
Subject: [PATCH 13/33] added resolver IP addresses to some log messages

---
 resolvers/pool.go     |  2 +-
 resolvers/resolver.go | 24 +++++++++++++++---------
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/resolvers/pool.go b/resolvers/pool.go
index e25ae8eb2..4d20a62de 100644
--- a/resolvers/pool.go
+++ b/resolvers/pool.go
@@ -338,7 +338,7 @@ func (rp *ResolverPool) performElection(votes []*resolveVote, name, qtype string
 	}
 
 	if len(ans) == 0 {
-		return ans, &ResolveError{Err: fmt.Sprintf("DNS query for %s, type %d returned 0 records", name, qt)}
+		return ans, &ResolveError{Err: fmt.Sprintf("Resolver Pool: DNS query for %s type %d returned 0 records", name, qt)}
 	}
 	return ans, nil
 }
diff --git a/resolvers/resolver.go b/resolvers/resolver.go
index 71ebfb7f6..cb3c3fa18 100644
--- a/resolvers/resolver.go
+++ b/resolvers/resolver.go
@@ -62,6 +62,7 @@ func randomInt(min, max int) int {
 type Resolver struct {
 	sync.RWMutex
 	Address        string
+	Port           string
 	WindowDuration time.Duration
 	Dialer         *net.Dialer
 	Conn           net.Conn
@@ -80,19 +81,22 @@ type Resolver struct {
 
 // NewResolver initializes a Resolver that send DNS queries to the IP address in the addr value.
 func NewResolver(addr string) *Resolver {
+	port := "53"
 	parts := strings.Split(addr, ":")
-	if len(parts) == 1 && parts[0] == addr {
-		addr += ":53"
+	if len(parts) == 2 {
+		addr = parts[0]
+		port = parts[1]
 	}
 
 	d := &net.Dialer{}
-	conn, err := d.Dial("udp", addr)
+	conn, err := d.Dial("udp", addr+":"+port)
 	if err != nil {
 		return nil
 	}
 
 	r := &Resolver{
 		Address:        addr,
+		Port:           port,
 		WindowDuration: 2 * time.Second,
 		Dialer:         d,
 		Conn:           conn,
@@ -339,7 +343,8 @@ func (r *Resolver) checkForTimeouts() {
 			// Remove the timed out requests from the map
 			for _, id := range timeouts {
 				if req := r.pullRequest(id); req != nil {
-					estr := fmt.Sprintf("DNS query for %s, type %d timed out", req.Name, req.Qtype)
+					estr := fmt.Sprintf("DNS query on resolver %s, for %s type %d timed out",
+						r.Address, req.Name, req.Qtype)
 					r.returnRequest(req, makeResolveResult(nil, true, estr, 100))
 				}
 			}
@@ -423,10 +428,10 @@ func (r *Resolver) tcpExchange(req *resolveRequest) {
 	msg := queryMessage(r.getID(), req.Name, req.Qtype)
 	d := net.Dialer{Timeout: r.WindowDuration}
 
-	conn, err := d.Dial("tcp", r.Address)
+	conn, err := d.Dial("tcp", r.Address+":"+r.Port)
 	if err != nil {
 		r.pullRequest(msg.MsgHdr.Id)
-		estr := fmt.Sprintf("DNS: Failed to obtain TCP connection to %s: %v", r.Address, err)
+		estr := fmt.Sprintf("DNS: Failed to obtain TCP connection to %s: %v", r.Address+":"+r.Port, err)
 		r.returnRequest(req, makeResolveResult(nil, true, estr, 100))
 		return
 	}
@@ -470,8 +475,8 @@ func (r *Resolver) processMessage(msg *dns.Msg) {
 				break
 			}
 		}
-		estr := fmt.Sprintf("DNS query for %s, type %d returned error %s",
-			req.Name, req.Qtype, dns.RcodeToString[msg.Rcode])
+		estr := fmt.Sprintf("DNS query on resolver %s, for %s type %d returned error %s",
+			r.Address, req.Name, req.Qtype, dns.RcodeToString[msg.Rcode])
 		r.returnRequest(req, makeResolveResult(nil, again, estr, msg.Rcode))
 		return
 	}
@@ -492,7 +497,8 @@ func (r *Resolver) processMessage(msg *dns.Msg) {
 	}
 
 	if len(answers) == 0 {
-		estr := fmt.Sprintf("DNS query for %s, type %d returned 0 records", req.Name, req.Qtype)
+		estr := fmt.Sprintf("DNS query on resolver %s, for %s type %d returned 0 records",
+			r.Address, req.Name, req.Qtype)
 		r.returnRequest(req, makeResolveResult(nil, false, estr, msg.Rcode))
 		return
 	}

From debafd8e9234bb06b867f011f006da9124863452 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Sun, 28 Jul 2019 20:26:35 -0400
Subject: [PATCH 14/33] added a new contributor

---
 README.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/README.md b/README.md
index 35a8a7547..07fe93e43 100644
--- a/README.md
+++ b/README.md
@@ -68,10 +68,12 @@ This project improves thanks to all the people who contribute:
 [![Follow on Twitter](https://img.shields.io/twitter/follow/sec_for_safety.svg?logo=twitter)](https://twitter.com/sec_for_safety)
 [![Follow on Twitter](https://img.shields.io/twitter/follow/ngkogkos.svg?logo=twitter)](https://github.com/ngkogkos)
 [![Follow on Twitter](https://img.shields.io/twitter/follow/Jhaddix.svg?logo=twitter)](https://twitter.com/Jhaddix)
+[![Follow on Twitter](https://img.shields.io/twitter/follow/Vltraheaven.svg?logo=twitter)](https://twitter.com/Vltraheaven)
 
 ## Mentions
 
 * [amass — Automated Attack Surface Mapping](https://danielmiessler.com/study/amass/)
+* [Aquatone — A Tool for Domain Flyovers](https://github.com/michenriksen/aquatone)
 * [Collaborating with the Crowd – Recapping LevelUp 0X04](https://www.bugcrowd.com/blog/recapping_levelup_0x04/)
 * [Subdomain Enumeration: 2019 Workflow](https://0xpatrik.com/subdomain-enumeration-2019/)
 * [REMOTE CODE EXECUTION ! 😜 Recon Wins](https://medium.com/@vishnu0002/remote-code-execution-recon-wins-e9c1db79f3da)

From 3dfba5fba467c946fb57ca4f35710a7bcb4da366 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Sun, 28 Jul 2019 20:28:03 -0400
Subject: [PATCH 15/33] added a log service to handle high concurrency

---
 enum/enum.go                       |   7 +-
 requests/request.go                |   1 +
 services/addrsrv.go                |   8 +--
 services/brute.go                  |   7 +-
 services/datamgmtsrv.go            |  27 ++++----
 services/dnssrv.go                 |  30 ++++-----
 services/logsrv.go                 |  71 +++++++++++++++++++
 services/markov.go                 |   4 +-
 services/sources/alienvault.go     |  42 +++++++-----
 services/sources/archiveit.go      |   8 ++-
 services/sources/archivetoday.go   |   8 ++-
 services/sources/arquivo.go        |   8 ++-
 services/sources/ask.go            |   7 +-
 services/sources/baidu.go          |  11 +--
 services/sources/binaryedge.go     |   9 +--
 services/sources/bing.go           |   7 +-
 services/sources/bufferover.go     |   6 +-
 services/sources/censys.go         |  12 ++--
 services/sources/certspotter.go    |   3 +-
 services/sources/circl.go          |   5 +-
 services/sources/commoncrawl.go    | 105 ++++++++++-------------------
 services/sources/crtsh.go          |   9 ++-
 services/sources/dnsdb.go          |   8 +--
 services/sources/dnsdumpster.go    |  11 +--
 services/sources/dnstable.go       |   2 +-
 services/sources/dogpile.go        |   3 +-
 services/sources/entrust.go        |   3 +-
 services/sources/exalead.go        |   2 +-
 services/sources/google.go         |   3 +-
 services/sources/googlect.go       |   3 +-
 services/sources/hackerone.go      |   2 +-
 services/sources/hackertarget.go   |  10 +--
 services/sources/ipv4info.go       |   8 +--
 services/sources/locarchive.go     |   4 +-
 services/sources/mnemonic.go       |   2 +-
 services/sources/netcraft.go       |   2 +-
 services/sources/networksdb.go     |  93 ++++++++++++++++---------
 services/sources/openukarchive.go  |   4 +-
 services/sources/passivetotal.go   |   5 +-
 services/sources/ptrarchive.go     |   2 +-
 services/sources/radb.go           |  36 ++++++----
 services/sources/riddler.go        |   2 +-
 services/sources/robtex.go         |  17 +++--
 services/sources/securitytrails.go |   6 +-
 services/sources/shadowserver.go   |  26 +++++--
 services/sources/shodan.go         |   6 +-
 services/sources/sitedossier.go    |   2 +-
 services/sources/spyse.go          |  18 +++--
 services/sources/sublist3r.go      |  10 +--
 services/sources/teamcymru.go      |  29 ++++++--
 services/sources/threatcrowd.go    |   6 +-
 services/sources/twitter.go        |   6 +-
 services/sources/ukgovarchive.go   |   8 ++-
 services/sources/umbrella.go       |  16 +++--
 services/sources/urlscan.go        |  10 +--
 services/sources/viewdns.go        |   8 ++-
 services/sources/virustotal.go     |  16 +++--
 services/sources/wayback.go        |  12 ++--
 services/sources/yahoo.go          |   7 +-
 59 files changed, 492 insertions(+), 311 deletions(-)
 create mode 100644 services/logsrv.go

diff --git a/enum/enum.go b/enum/enum.go
index d0f7a6675..0515aa2d2 100644
--- a/enum/enum.go
+++ b/enum/enum.go
@@ -273,12 +273,15 @@ func (e *Enumeration) requiredServices() []services.Service {
 		if e.Config.DataOptsWriter != nil {
 			dms.AddDataHandler(graph.NewDataOptsHandler(e.Config.DataOptsWriter))
 		}
-		srvcs = append(srvcs, services.NewDNSService(e.Config, e.Bus, e.Pool), dms)
+		srvcs = append(srvcs, dms, services.NewDNSService(e.Config, e.Bus, e.Pool))
 	}
 
 	namesrv := services.NewNameService(e.Config, e.Bus, e.Pool)
 	namesrv.RegisterGraph(e.Graph)
-	srvcs = append(srvcs, namesrv, services.NewAddressService(e.Config, e.Bus, e.Pool))
+	srvcs = append(srvcs, namesrv,
+		services.NewLogService(e.Config, e.Bus, e.Pool),
+		services.NewAddressService(e.Config, e.Bus, e.Pool),
+	)
 
 	if !e.Config.Passive {
 		e.bruteSrv = services.NewBruteForceService(e.Config, e.Bus, e.Pool)
diff --git a/requests/request.go b/requests/request.go
index 8830cd39d..4bcf55ae8 100644
--- a/requests/request.go
+++ b/requests/request.go
@@ -34,6 +34,7 @@ const (
 	NewASNTopic       = "amass:asn"
 	WhoisRequestTopic = "amass:whoisreq"
 	NewWhoisTopic     = "amass:whoisinfo"
+	LogTopic          = "amass:log"
 )
 
 // DNSAnswer is the type used by Amass to represent a DNS record.
diff --git a/services/addrsrv.go b/services/addrsrv.go
index a8429465f..e170d51f8 100644
--- a/services/addrsrv.go
+++ b/services/addrsrv.go
@@ -21,6 +21,7 @@ var (
 	// Cache for the infrastructure data collected from online sources
 	netLock  sync.Mutex
 	netCache map[int]*requests.ASNRequest
+
 	// The reserved network address ranges
 	reservedAddrRanges []*net.IPNet
 	reservedCIDRs      = []string{
@@ -45,8 +46,7 @@ var (
 	}
 )
 
-// AddressService is the Service that handles all newly discovered IP addresses
-// within the architecture. This is achieved by receiving all the NEWADDR events.
+// AddressService is the Service that handles all newly discovered IP addresses within the architecture.
 type AddressService struct {
 	BaseService
 
@@ -64,10 +64,10 @@ func init() {
 }
 
 // NewAddressService returns he object initialized, but not yet started.
-func NewAddressService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AddressService {
+func NewAddressService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AddressService {
 	as := &AddressService{filter: utils.NewStringFilter()}
 
-	as.BaseService = *NewBaseService(as, "Address Service", c, bus, pool)
+	as.BaseService = *NewBaseService(as, "Address Service", cfg, bus, pool)
 	return as
 }
 
diff --git a/services/brute.go b/services/brute.go
index 0eb00a092..83a44d3b2 100644
--- a/services/brute.go
+++ b/services/brute.go
@@ -4,6 +4,7 @@
 package services
 
 import (
+	"fmt"
 	"strings"
 	"sync"
 	"time"
@@ -37,10 +38,10 @@ type BruteForceService struct {
 }
 
 // NewBruteForceService returns he object initialized, but not yet started.
-func NewBruteForceService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BruteForceService {
+func NewBruteForceService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BruteForceService {
 	bfs := &BruteForceService{filter: utils.NewStringFilter()}
 
-	bfs.BaseService = *NewBaseService(bfs, "Brute Forcing", c, bus, pool)
+	bfs.BaseService = *NewBaseService(bfs, "Brute Forcing", cfg, bus, pool)
 	return bfs
 }
 
@@ -190,7 +191,7 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) {
 				break
 			}
 		} else {
-			bfs.Config().Log.Printf("%s: %v", bfs.String(), err)
+			bfs.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", bfs.String(), err))
 		}
 		bfs.metrics.QueryTime(time.Now())
 		bfs.SetActive()
diff --git a/services/datamgmtsrv.go b/services/datamgmtsrv.go
index 2a0b85fe0..c32b06089 100644
--- a/services/datamgmtsrv.go
+++ b/services/datamgmtsrv.go
@@ -4,6 +4,7 @@
 package services
 
 import (
+	"fmt"
 	"regexp"
 	"strings"
 	"time"
@@ -27,10 +28,10 @@ type DataManagerService struct {
 }
 
 // NewDataManagerService returns he object initialized, but not yet started.
-func NewDataManagerService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DataManagerService {
+func NewDataManagerService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DataManagerService {
 	dms := &DataManagerService{domainFilter: utils.NewStringFilter()}
 
-	dms.BaseService = *NewBaseService(dms, "Data Manager", c, bus, pool)
+	dms.BaseService = *NewBaseService(dms, "Data Manager", cfg, bus, pool)
 	return dms
 }
 
@@ -116,7 +117,7 @@ func (dms *DataManagerService) insertDomain(domain string) {
 			Source:    "Forward DNS",
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert domain: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert domain: %v", handler, err))
 		}
 	}
 	dms.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
@@ -150,7 +151,7 @@ func (dms *DataManagerService) insertCNAME(req *requests.DNSRequest, recidx int)
 			Source:       req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert CNAME: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert CNAME: %v", handler, err))
 		}
 	}
 	dms.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
@@ -178,7 +179,7 @@ func (dms *DataManagerService) insertA(req *requests.DNSRequest, recidx int) {
 			Source:    req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert A record: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert A record: %v", handler, err))
 		}
 	}
 	dms.insertInfrastructure(addr)
@@ -209,7 +210,7 @@ func (dms *DataManagerService) insertAAAA(req *requests.DNSRequest, recidx int)
 			Source:    req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert AAAA record: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert AAAA record: %v", handler, err))
 		}
 	}
 	dms.insertInfrastructure(addr)
@@ -245,7 +246,7 @@ func (dms *DataManagerService) insertPTR(req *requests.DNSRequest, recidx int) {
 			Source:     req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert PTR record: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert PTR record: %v", handler, err))
 		}
 	}
 	dms.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
@@ -276,7 +277,7 @@ func (dms *DataManagerService) insertSRV(req *requests.DNSRequest, recidx int) {
 			Source:     req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert SRV record: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert SRV record: %v", handler, err))
 		}
 	}
 
@@ -314,7 +315,7 @@ func (dms *DataManagerService) insertNS(req *requests.DNSRequest, recidx int) {
 			Source:       req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert NS record: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert NS record: %v", handler, err))
 		}
 	}
 	if target != domain {
@@ -350,7 +351,7 @@ func (dms *DataManagerService) insertMX(req *requests.DNSRequest, recidx int) {
 			Source:       req.Source,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s failed to insert MX record: %v", handler, err)
+			dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s failed to insert MX record: %v", handler, err))
 		}
 	}
 	if target != domain {
@@ -409,7 +410,7 @@ func (dms *DataManagerService) findNamesAndAddresses(data, domain string) {
 func (dms *DataManagerService) insertInfrastructure(addr string) {
 	asn, cidr, desc, err := IPRequest(addr, dms.Bus())
 	if err != nil {
-		dms.Config().Log.Printf("%s: %v", dms.String(), err)
+		dms.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", dms.String(), err))
 		return
 	}
 
@@ -424,7 +425,9 @@ func (dms *DataManagerService) insertInfrastructure(addr string) {
 			Description: desc,
 		})
 		if err != nil {
-			dms.Config().Log.Printf("%s: %s failed to insert infrastructure data: %v", dms.String(), handler, err)
+			dms.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: %s failed to insert infrastructure data: %v", dms.String(), handler, err),
+			)
 		}
 	}
 }
diff --git a/services/dnssrv.go b/services/dnssrv.go
index 430f3955d..dcaedab2c 100644
--- a/services/dnssrv.go
+++ b/services/dnssrv.go
@@ -49,7 +49,7 @@ type DNSService struct {
 }
 
 // NewDNSService returns he object initialized, but not yet started.
-func NewDNSService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DNSService {
+func NewDNSService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *DNSService {
 	ds := &DNSService{filter: utils.NewStringFilter()}
 
 	for _, n := range badSubnets {
@@ -58,7 +58,7 @@ func NewDNSService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverP
 		}
 	}
 
-	ds.BaseService = *NewBaseService(ds, "DNS Service", c, bus, pool)
+	ds.BaseService = *NewBaseService(ds, "DNS Service", cfg, bus, pool)
 	return ds
 }
 
@@ -179,7 +179,7 @@ func (ds *DNSService) performDNSRequest(req *requests.DNSRequest) {
 				break
 			}
 		} else {
-			ds.Config().Log.Printf("DNS: %v", err)
+			ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: %v", err))
 		}
 		ds.metrics.QueryTime(time.Now())
 		ds.SetActive()
@@ -246,7 +246,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) {
 			answers = append(answers, a)
 		}
 	} else {
-		ds.Config().Log.Printf("DNS: NS record query error: %s: %v", subdomain, err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: NS record query error: %s: %v", subdomain, err))
 	}
 	ds.metrics.QueryTime(time.Now())
 
@@ -257,7 +257,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) {
 			answers = append(answers, a)
 		}
 	} else {
-		ds.Config().Log.Printf("DNS: MX record query error: %s: %v", subdomain, err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: MX record query error: %s: %v", subdomain, err))
 	}
 	ds.metrics.QueryTime(time.Now())
 
@@ -266,7 +266,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) {
 	if ans, err := ds.Pool().Resolve(subdomain, "SOA", resolvers.PriorityHigh); err == nil {
 		answers = append(answers, ans...)
 	} else {
-		ds.Config().Log.Printf("DNS: SOA record query error: %s: %v", subdomain, err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: SOA record query error: %s: %v", subdomain, err))
 	}
 	ds.metrics.QueryTime(time.Now())
 
@@ -275,7 +275,7 @@ func (ds *DNSService) basicQueries(subdomain, domain string) {
 	if ans, err := ds.Pool().Resolve(subdomain, "SPF", resolvers.PriorityHigh); err == nil {
 		answers = append(answers, ans...)
 	} else {
-		ds.Config().Log.Printf("DNS: SPF record query error: %s: %v", subdomain, err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: SPF record query error: %s: %v", subdomain, err))
 	}
 	ds.metrics.QueryTime(time.Now())
 
@@ -298,17 +298,17 @@ func (ds *DNSService) attemptZoneXFR(sub, domain, server string) {
 
 	addr, err := ds.nameserverAddr(server)
 	if addr == "" {
-		ds.Config().Log.Printf("DNS: Zone XFR failed: %v", err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone XFR failed: %v", err))
 		return
 	}
 
-	requests, err := resolvers.ZoneTransfer(sub, domain, addr)
+	reqs, err := resolvers.ZoneTransfer(sub, domain, addr)
 	if err != nil {
-		ds.Config().Log.Printf("DNS: Zone XFR failed: %s: %v", server, err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone XFR failed: %s: %v", server, err))
 		return
 	}
 
-	for _, req := range requests {
+	for _, req := range reqs {
 		ds.resolvedName(req)
 	}
 }
@@ -316,17 +316,17 @@ func (ds *DNSService) attemptZoneXFR(sub, domain, server string) {
 func (ds *DNSService) attemptZoneWalk(domain, server string) {
 	addr, err := ds.nameserverAddr(server)
 	if addr == "" {
-		ds.Config().Log.Printf("DNS: Zone Walk failed: %v", err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone Walk failed: %v", err))
 		return
 	}
 
-	requests, err := resolvers.NsecTraversal(domain, addr)
+	reqs, err := resolvers.NsecTraversal(domain, addr)
 	if err != nil {
-		ds.Config().Log.Printf("DNS: Zone Walk failed: %s: %v", server, err)
+		ds.Bus().Publish(requests.LogTopic, fmt.Sprintf("DNS: Zone Walk failed: %s: %v", server, err))
 		return
 	}
 
-	for _, req := range requests {
+	for _, req := range reqs {
 		ds.SendDNSRequest(req)
 	}
 }
diff --git a/services/logsrv.go b/services/logsrv.go
new file mode 100644
index 000000000..3cffbac54
--- /dev/null
+++ b/services/logsrv.go
@@ -0,0 +1,71 @@
+// Copyright 2017 Jeff Foley. All rights reserved.
+// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+
+package services
+
+import (
+	"time"
+
+	"github.com/OWASP/Amass/config"
+	eb "github.com/OWASP/Amass/eventbus"
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+	"github.com/OWASP/Amass/utils"
+)
+
+// LogService is the Service that performs logging for the architecture.
+type LogService struct {
+	BaseService
+
+	queue *utils.Queue
+}
+
+// NewLogService returns he object initialized, but not yet started.
+func NewLogService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *LogService {
+	l := &LogService{queue: new(utils.Queue)}
+
+	l.BaseService = *NewBaseService(l, "Log Service", cfg, bus, pool)
+	return l
+}
+
+// OnStart implements the Service interface.
+func (l *LogService) OnStart() error {
+	l.BaseService.OnStart()
+
+	l.Bus().Subscribe(requests.LogTopic, l.queue.Append)
+	go l.processRequests()
+	return nil
+}
+
+func (l *LogService) processRequests() {
+	t := time.NewTicker(2 * time.Second)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-l.PauseChan():
+			<-l.ResumeChan()
+		case <-l.Quit():
+			return
+		case <-t.C:
+			if !l.queue.Empty() {
+				l.writeLogs()
+			}
+		case <-l.DNSRequestChan():
+		case <-l.AddrRequestChan():
+		case <-l.ASNRequestChan():
+		case <-l.WhoisRequestChan():
+		}
+	}
+}
+
+func (l *LogService) writeLogs() {
+	for {
+		msg, ok := l.queue.Next()
+		if !ok {
+			break
+		}
+
+		l.Config().Log.Print(msg.(string))
+	}
+}
diff --git a/services/markov.go b/services/markov.go
index 1da064006..4e385eab3 100644
--- a/services/markov.go
+++ b/services/markov.go
@@ -55,7 +55,7 @@ type MarkovService struct {
 }
 
 // NewMarkovService returns he object initialized, but not yet started.
-func NewMarkovService(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *MarkovService {
+func NewMarkovService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *MarkovService {
 	m := &MarkovService{
 		subs:      make(map[string]*requests.DNSRequest),
 		inFilter:  utils.NewStringFilter(),
@@ -66,7 +66,7 @@ func NewMarkovService(c *config.Config, bus *eb.EventBus, pool *resolvers.Resolv
 		},
 	}
 
-	m.BaseService = *NewBaseService(m, "Markov Model", c, bus, pool)
+	m.BaseService = *NewBaseService(m, "Markov Model", cfg, bus, pool)
 	return m
 }
 
diff --git a/services/sources/alienvault.go b/services/sources/alienvault.go
index 4b41531b6..e39a67959 100644
--- a/services/sources/alienvault.go
+++ b/services/sources/alienvault.go
@@ -29,13 +29,13 @@ type AlienVault struct {
 }
 
 // NewAlienVault returns he object initialized, but not yet started.
-func NewAlienVault(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AlienVault {
+func NewAlienVault(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AlienVault {
 	a := &AlienVault{
 		SourceType: requests.API,
 		RateLimit:  100 * time.Millisecond,
 	}
 
-	a.BaseService = *services.NewBaseService(a, "AlienVault", c, bus, pool)
+	a.BaseService = *services.NewBaseService(a, "AlienVault", cfg, bus, pool)
 	return a
 }
 
@@ -45,7 +45,7 @@ func (a *AlienVault) OnStart() error {
 
 	a.API = a.Config().GetAPIKey(a.String())
 	if a.API == nil || a.API.Key == "" {
-		a.Config().Log.Printf("%s: API key data was not provided", a.String())
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", a.String()))
 	}
 
 	go a.processRequests()
@@ -95,7 +95,7 @@ func (a *AlienVault) executeDNSQuery(domain string) {
 	u := a.getURL(domain) + "passive_dns"
 	page, err := utils.RequestWebPage(u, nil, a.getHeaders(), "", "")
 	if err != nil {
-		a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 		return
 	}
 	// Extract the subdomain names and IP addresses from the passive DNS information
@@ -106,10 +106,10 @@ func (a *AlienVault) executeDNSQuery(domain string) {
 		} `json:"passive_dns"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 		return
 	} else if len(m.Subdomains) == 0 {
-		a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), u)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u))
 		return
 	}
 
@@ -153,7 +153,7 @@ func (a *AlienVault) executeURLQuery(domain string) {
 	u := a.getURL(domain) + "url_list"
 	page, err := utils.RequestWebPage(u, nil, headers, "", "")
 	if err != nil {
-		a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 		return
 	}
 	// Extract the subdomain names and IP addresses from the URL information
@@ -173,10 +173,10 @@ func (a *AlienVault) executeURLQuery(domain string) {
 		} `json:"url_list"`
 	}
 	if err := json.Unmarshal([]byte(page), &urls); err != nil {
-		a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 		return
 	} else if len(urls.URLs) == 0 {
-		a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), u)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u))
 		return
 	}
 
@@ -200,15 +200,17 @@ func (a *AlienVault) executeURLQuery(domain string) {
 			pageURL := u + "?page=" + strconv.Itoa(cur)
 			page, err = utils.RequestWebPage(pageURL, nil, headers, "", "")
 			if err != nil {
-				a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err)
+				a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err))
 				break
 			}
 
 			if err := json.Unmarshal([]byte(page), &urls); err != nil {
-				a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err)
+				a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err))
 				break
 			} else if len(urls.URLs) == 0 {
-				a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), pageURL)
+				a.Bus().Publish(requests.LogTopic,
+					fmt.Sprintf("%s: %s: The query returned zero results", a.String(), pageURL),
+				)
 				break
 			}
 
@@ -250,7 +252,7 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 	a.SetActive()
 	page, err := utils.RequestWebPage(u, nil, a.getHeaders(), "", "")
 	if err != nil {
-		a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 		return emails
 	}
 
@@ -263,10 +265,12 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 		} `json:"data"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 		return emails
 	} else if m.Count == 0 {
-		a.Config().Log.Printf("%s: %s: The query returned zero results", a.String(), u)
+		a.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u),
+		)
 		return emails
 	}
 
@@ -300,7 +304,7 @@ func (a *AlienVault) executeWhoisQuery(domain string) {
 		pageURL := a.getReverseWhoisURL(email)
 		page, err := utils.RequestWebPage(pageURL, nil, headers, "", "")
 		if err != nil {
-			a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err)
+			a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err))
 			continue
 		}
 
@@ -309,7 +313,7 @@ func (a *AlienVault) executeWhoisQuery(domain string) {
 		}
 		var domains []record
 		if err := json.Unmarshal([]byte(page), &domains); err != nil {
-			a.Config().Log.Printf("%s: %s: %v", a.String(), pageURL, err)
+			a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), pageURL, err))
 			continue
 		}
 		for _, d := range domains {
@@ -321,7 +325,9 @@ func (a *AlienVault) executeWhoisQuery(domain string) {
 	}
 
 	if len(newDomains) == 0 {
-		a.Config().Log.Printf("%s: Reverse whois failed to discover new domain names for %s", a.String(), domain)
+		a.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Reverse whois failed to discover new domain names for %s", a.String(), domain),
+		)
 		return
 	}
 
diff --git a/services/sources/archiveit.go b/services/sources/archiveit.go
index 0e4824bd6..edb17cf68 100644
--- a/services/sources/archiveit.go
+++ b/services/sources/archiveit.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -23,7 +25,7 @@ type ArchiveIt struct {
 }
 
 // NewArchiveIt returns he object initialized, but not yet started.
-func NewArchiveIt(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveIt {
+func NewArchiveIt(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveIt {
 	a := &ArchiveIt{
 		domain:     "wayback.archive-it.org",
 		baseURL:    "https://wayback.archive-it.org/all",
@@ -31,7 +33,7 @@ func NewArchiveIt(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPo
 		filter:     utils.NewStringFilter(),
 	}
 
-	a.BaseService = *services.NewBaseService(a, "ArchiveIt", c, bus, pool)
+	a.BaseService = *services.NewBaseService(a, "ArchiveIt", cfg, bus, pool)
 	return a
 }
 
@@ -67,7 +69,7 @@ func (a *ArchiveIt) executeQuery(sn, domain string) {
 
 	names, err := crawl(a, a.baseURL, a.domain, sn, domain)
 	if err != nil {
-		a.Config().Log.Printf("%s: %v", a.String(), err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", a.String(), err))
 		return
 	}
 
diff --git a/services/sources/archivetoday.go b/services/sources/archivetoday.go
index ade2f8634..8d53e2b22 100644
--- a/services/sources/archivetoday.go
+++ b/services/sources/archivetoday.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -23,7 +25,7 @@ type ArchiveToday struct {
 }
 
 // NewArchiveToday returns he object initialized, but not yet started.
-func NewArchiveToday(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveToday {
+func NewArchiveToday(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *ArchiveToday {
 	a := &ArchiveToday{
 		domain:     "archive.is",
 		baseURL:    "http://archive.is",
@@ -31,7 +33,7 @@ func NewArchiveToday(c *config.Config, bus *eb.EventBus, pool *resolvers.Resolve
 		filter:     utils.NewStringFilter(),
 	}
 
-	a.BaseService = *services.NewBaseService(a, "ArchiveToday", c, bus, pool)
+	a.BaseService = *services.NewBaseService(a, "ArchiveToday", cfg, bus, pool)
 	return a
 }
 
@@ -67,7 +69,7 @@ func (a *ArchiveToday) executeQuery(sn, domain string) {
 
 	names, err := crawl(a, a.baseURL, a.domain, sn, domain)
 	if err != nil {
-		a.Config().Log.Printf("%s: %v", a.String(), err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", a.String(), err))
 		return
 	}
 
diff --git a/services/sources/arquivo.go b/services/sources/arquivo.go
index 8963b40ce..b9399985d 100644
--- a/services/sources/arquivo.go
+++ b/services/sources/arquivo.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -23,7 +25,7 @@ type Arquivo struct {
 }
 
 // NewArquivo returns he object initialized, but not yet started.
-func NewArquivo(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Arquivo {
+func NewArquivo(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Arquivo {
 	a := &Arquivo{
 		domain:     "arquivo.pt",
 		baseURL:    "http://arquivo.pt/wayback",
@@ -31,7 +33,7 @@ func NewArquivo(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool
 		filter:     utils.NewStringFilter(),
 	}
 
-	a.BaseService = *services.NewBaseService(a, "Arquivo", c, bus, pool)
+	a.BaseService = *services.NewBaseService(a, "Arquivo", cfg, bus, pool)
 	return a
 }
 
@@ -67,7 +69,7 @@ func (a *Arquivo) executeQuery(sn, domain string) {
 
 	names, err := crawl(a, a.baseURL, a.domain, sn, domain)
 	if err != nil {
-		a.Config().Log.Printf("%s: %v", a.String(), err)
+		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", a.String(), err))
 		return
 	}
 
diff --git a/services/sources/ask.go b/services/sources/ask.go
index 9ba24bd7f..b848b8a1c 100644
--- a/services/sources/ask.go
+++ b/services/sources/ask.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"time"
@@ -26,14 +27,14 @@ type Ask struct {
 }
 
 // NewAsk returns he object initialized, but not yet started.
-func NewAsk(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Ask {
+func NewAsk(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Ask {
 	a := &Ask{
 		quantity:   10, // ask.com appears to be hardcoded at 10 results per page
 		limit:      100,
 		SourceType: requests.SCRAPE,
 	}
 
-	a.BaseService = *services.NewBaseService(a, "Ask", c, bus, pool)
+	a.BaseService = *services.NewBaseService(a, "Ask", cfg, bus, pool)
 	return a
 }
 
@@ -81,7 +82,7 @@ func (a *Ask) executeQuery(domain string) {
 			u := a.urlByPageNum(domain, i)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				a.Config().Log.Printf("%s: %s: %v", a.String(), u, err)
+				a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
 				return
 			}
 
diff --git a/services/sources/baidu.go b/services/sources/baidu.go
index d24613521..74f0be501 100644
--- a/services/sources/baidu.go
+++ b/services/sources/baidu.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"time"
@@ -20,20 +21,20 @@ import (
 type Baidu struct {
 	services.BaseService
 
+	SourceType string
 	quantity   int
 	limit      int
-	SourceType string
 }
 
 // NewBaidu returns he object initialized, but not yet started.
-func NewBaidu(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Baidu {
+func NewBaidu(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Baidu {
 	b := &Baidu{
+		SourceType: requests.SCRAPE,
 		quantity:   20,
 		limit:      100,
-		SourceType: requests.SCRAPE,
 	}
 
-	b.BaseService = *services.NewBaseService(b, "Baidu", c, bus, pool)
+	b.BaseService = *services.NewBaseService(b, "Baidu", cfg, bus, pool)
 	return b
 }
 
@@ -81,7 +82,7 @@ func (b *Baidu) executeQuery(domain string) {
 			u := b.urlByPageNum(domain, i)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				b.Config().Log.Printf("%s: %s: %v", b.String(), u, err)
+				b.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", b.String(), u, err))
 				return
 			}
 
diff --git a/services/sources/binaryedge.go b/services/sources/binaryedge.go
index eeb5c9609..95c6ce505 100644
--- a/services/sources/binaryedge.go
+++ b/services/sources/binaryedge.go
@@ -5,6 +5,7 @@ package sources
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/OWASP/Amass/config"
@@ -25,13 +26,13 @@ type BinaryEdge struct {
 }
 
 // NewBinaryEdge returns he object initialized, but not yet started.
-func NewBinaryEdge(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BinaryEdge {
+func NewBinaryEdge(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BinaryEdge {
 	be := &BinaryEdge{
 		SourceType: requests.API,
 		RateLimit:  2 * time.Second,
 	}
 
-	be.BaseService = *services.NewBaseService(be, "BinaryEdge", c, bus, pool)
+	be.BaseService = *services.NewBaseService(be, "BinaryEdge", cfg, bus, pool)
 	return be
 }
 
@@ -41,7 +42,7 @@ func (be *BinaryEdge) OnStart() error {
 
 	be.API = be.Config().GetAPIKey(be.String())
 	if be.API == nil || be.API.Key == "" {
-		be.Config().Log.Printf("%s: API key data was not provided", be.String())
+		be.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", be.String()))
 	}
 
 	go be.processRequests()
@@ -90,7 +91,7 @@ func (be *BinaryEdge) executeQuery(domain string) {
 	be.SetActive()
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		be.Config().Log.Printf("%s: %s: %v", be.String(), url, err)
+		be.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", be.String(), url, err))
 		return
 	}
 	// Extract the subdomain names from the REST API results
diff --git a/services/sources/bing.go b/services/sources/bing.go
index 5ea0f422a..9834834bd 100644
--- a/services/sources/bing.go
+++ b/services/sources/bing.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"time"
@@ -26,14 +27,14 @@ type Bing struct {
 }
 
 // NewBing returns he object initialized, but not yet started.
-func NewBing(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Bing {
+func NewBing(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Bing {
 	b := &Bing{
 		quantity:   20,
 		limit:      200,
 		SourceType: requests.SCRAPE,
 	}
 
-	b.BaseService = *services.NewBaseService(b, "Bing", c, bus, pool)
+	b.BaseService = *services.NewBaseService(b, "Bing", cfg, bus, pool)
 	return b
 }
 
@@ -81,7 +82,7 @@ func (b *Bing) executeQuery(domain string) {
 			u := b.urlByPageNum(domain, i)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				b.Config().Log.Printf("%s: %s: %v", b.String(), u, err)
+				b.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", b.String(), u, err))
 				return
 			}
 
diff --git a/services/sources/bufferover.go b/services/sources/bufferover.go
index b550b087c..f2a941beb 100644
--- a/services/sources/bufferover.go
+++ b/services/sources/bufferover.go
@@ -22,10 +22,10 @@ type BufferOver struct {
 }
 
 // NewBufferOver returns he object initialized, but not yet started.
-func NewBufferOver(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BufferOver {
+func NewBufferOver(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *BufferOver {
 	b := &BufferOver{SourceType: requests.API}
 
-	b.BaseService = *services.NewBaseService(b, "BufferOver", c, bus, pool)
+	b.BaseService = *services.NewBaseService(b, "BufferOver", cfg, bus, pool)
 	return b
 }
 
@@ -63,7 +63,7 @@ func (b *BufferOver) executeQuery(domain string) {
 	url := b.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		b.Config().Log.Printf("%s: %s: %v", b.String(), url, err)
+		b.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", b.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/censys.go b/services/sources/censys.go
index 61e4313af..1d95173e6 100644
--- a/services/sources/censys.go
+++ b/services/sources/censys.go
@@ -44,7 +44,7 @@ func (c *Censys) OnStart() error {
 
 	c.API = c.Config().GetAPIKey(c.String())
 	if c.API == nil || c.API.Key == "" || c.API.Secret == "" {
-		c.Config().Log.Printf("%s: API key data was not provided", c.String())
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", c.String()))
 	}
 
 	go c.processRequests()
@@ -101,7 +101,7 @@ func (c *Censys) apiQuery(domain string) {
 		headers := map[string]string{"Content-Type": "application/json"}
 		resp, err := utils.RequestWebPage(u, body, headers, c.API.Key, c.API.Secret)
 		if err != nil {
-			c.Config().Log.Printf("%s: %s: %v", c.String(), u, err)
+			c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), u, err))
 			break
 		}
 		// Extract the subdomain names from the certificate information
@@ -116,10 +116,12 @@ func (c *Censys) apiQuery(domain string) {
 			} `json:"results"`
 		}
 		if err := json.Unmarshal([]byte(resp), &m); err != nil || m.Status != "ok" {
-			c.Config().Log.Printf("%s: %s: %v", c.String(), u, err)
+			c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), u, err))
 			break
 		} else if len(m.Results) == 0 {
-			c.Config().Log.Printf("%s: %s: The query returned zero results", c.String(), u)
+			c.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: %s: The query returned zero results", c.String(), u),
+			)
 			break
 		}
 
@@ -163,7 +165,7 @@ func (c *Censys) executeQuery(domain string) {
 	url = c.webURL(domain)
 	page, err = utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		c.Config().Log.Printf("%s: %s: %v", c.String(), url, err)
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/certspotter.go b/services/sources/certspotter.go
index 326e6bb83..f11e0646b 100644
--- a/services/sources/certspotter.go
+++ b/services/sources/certspotter.go
@@ -5,6 +5,7 @@ package sources
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/url"
 	"time"
 
@@ -76,7 +77,7 @@ func (c *CertSpotter) executeQuery(domain string) {
 	url := c.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		c.Config().Log.Printf("%s: %s: %v", c.String(), url, err)
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err))
 		return
 	}
 	// Extract the subdomain names from the certificate information
diff --git a/services/sources/circl.go b/services/sources/circl.go
index da4f458b1..f2d5cdb94 100644
--- a/services/sources/circl.go
+++ b/services/sources/circl.go
@@ -6,6 +6,7 @@ package sources
 import (
 	"bufio"
 	"encoding/json"
+	"fmt"
 	"strings"
 	"time"
 
@@ -43,7 +44,7 @@ func (c *CIRCL) OnStart() error {
 
 	c.API = c.Config().GetAPIKey(c.String())
 	if c.API == nil || c.API.Username == "" || c.API.Password == "" {
-		c.Config().Log.Printf("%s: API key data was not provided", c.String())
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", c.String()))
 	}
 
 	go c.processRequests()
@@ -83,7 +84,7 @@ func (c *CIRCL) executeQuery(domain string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, c.API.Username, c.API.Password)
 	if err != nil {
-		c.Config().Log.Printf("%s: %s: %v", c.String(), url, err)
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/commoncrawl.go b/services/sources/commoncrawl.go
index 4939b0f00..23f9d06d5 100644
--- a/services/sources/commoncrawl.go
+++ b/services/sources/commoncrawl.go
@@ -6,6 +6,7 @@ package sources
 import (
 	"bufio"
 	"encoding/json"
+	"fmt"
 	"net/url"
 	"strings"
 	"time"
@@ -18,84 +19,19 @@ import (
 	"github.com/OWASP/Amass/utils"
 )
 
-var (
-	commonCrawlIndexes = []string{
-		"CC-MAIN-2013-20",
-		"CC-MAIN-2013-48",
-		"CC-MAIN-2014-10",
-		"CC-MAIN-2014-15",
-		"CC-MAIN-2014-23",
-		"CC-MAIN-2014-35",
-		"CC-MAIN-2014-41",
-		"CC-MAIN-2014-42",
-		"CC-MAIN-2014-49",
-		"CC-MAIN-2014-52",
-		"CC-MAIN-2015-06",
-		"CC-MAIN-2015-11",
-		"CC-MAIN-2015-14",
-		"CC-MAIN-2015-18",
-		"CC-MAIN-2015-22",
-		"CC-MAIN-2015-27",
-		"CC-MAIN-2015-32",
-		"CC-MAIN-2015-35",
-		"CC-MAIN-2015-40",
-		"CC-MAIN-2015-48",
-		"CC-MAIN-2016-07",
-		"CC-MAIN-2016-18",
-		"CC-MAIN-2016-22",
-		"CC-MAIN-2016-26",
-		"CC-MAIN-2016-30",
-		"CC-MAIN-2016-36",
-		"CC-MAIN-2016-40",
-		"CC-MAIN-2016-44",
-		"CC-MAIN-2016-50",
-		"CC-MAIN-2017-04",
-		"CC-MAIN-2017-09",
-		"CC-MAIN-2017-13",
-		"CC-MAIN-2017-17",
-		"CC-MAIN-2017-22",
-		"CC-MAIN-2017-26",
-		"CC-MAIN-2017-30",
-		"CC-MAIN-2017-34",
-		"CC-MAIN-2017-39",
-		"CC-MAIN-2017-43",
-		"CC-MAIN-2017-47",
-		"CC-MAIN-2017-51",
-		"CC-MAIN-2018-05",
-		"CC-MAIN-2018-09",
-		"CC-MAIN-2018-13",
-		"CC-MAIN-2018-17",
-		"CC-MAIN-2018-22",
-		"CC-MAIN-2018-26",
-		"CC-MAIN-2018-30",
-		"CC-MAIN-2018-34",
-		"CC-MAIN-2018-39",
-		"CC-MAIN-2018-43",
-		"CC-MAIN-2018-47",
-		"CC-MAIN-2018-51",
-		"CC-MAIN-2019-04",
-		"CC-MAIN-2019-09",
-		"CC-MAIN-2019-13",
-		"CC-MAIN-2019-18",
-		"CC-MAIN-2019-22",
-		"CC-MAIN-2019-26",
-	}
-)
+const commonCrawlIndexListURL = "https://index.commoncrawl.org/collinfo.json"
 
 // CommonCrawl is the Service that handles access to the CommonCrawl data source.
 type CommonCrawl struct {
 	services.BaseService
 
-	baseURL    string
 	SourceType string
+	indexURLs  []string
 }
 
 // NewCommonCrawl returns he object initialized, but not yet started.
 func NewCommonCrawl(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *CommonCrawl {
-	c := &CommonCrawl{
-		baseURL:    "http://index.commoncrawl.org/",
-		SourceType: requests.API,
-	}
+	c := &CommonCrawl{SourceType: requests.API}
 
 	c.BaseService = *services.NewBaseService(c, "CommonCrawl", cfg, bus, pool)
 	return c
@@ -105,6 +41,33 @@ func NewCommonCrawl(cfg *config.Config, bus *eb.EventBus, pool *resolvers.Resolv
 func (c *CommonCrawl) OnStart() error {
 	c.BaseService.OnStart()
 
+	// Get all of the index API URLs
+	page, err := utils.RequestWebPage(commonCrawlIndexListURL, nil, nil, "", "")
+	if err != nil {
+		c.Bus().Publish(requests.LogTopic, 
+			fmt.Sprintf("%s: Failed to obtain the index list: %v", c.String(), err),
+		)
+		return fmt.Errorf("%s: Failed to obtain the index list: %v", c.String(), err)
+	}
+
+	type index struct {
+		ID   string `json:"id"`
+		Name string `json:"name"`
+		URL  string `json:"cdx-api"`
+	}
+
+	var indexList []index
+	if err := json.Unmarshal([]byte(page), &indexList); err != nil {
+		c.Bus().Publish(requests.LogTopic, 
+			fmt.Sprintf("%s: Failed to unmarshal the index list: %v", c.String(), err),
+		)
+		return fmt.Errorf("%s: Failed to unmarshal the index list: %v", c.String(), err)
+	}
+
+	for _, i := range indexList {
+		c.indexURLs = append(c.indexURLs, i.URL)
+	}
+
 	go c.processRequests()
 	return nil
 }
@@ -135,7 +98,7 @@ func (c *CommonCrawl) executeQuery(domain string) {
 	t := time.NewTicker(500 * time.Millisecond)
 	defer t.Stop()
 
-	for _, index := range commonCrawlIndexes {
+	for _, index := range c.indexURLs {
 		c.SetActive()
 
 		select {
@@ -145,7 +108,7 @@ func (c *CommonCrawl) executeQuery(domain string) {
 			u := c.getURL(domain, index)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				c.Config().Log.Printf("%s: %s: %v", c.String(), u, err)
+				c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), u, err))
 				continue
 			}
 
@@ -191,7 +154,7 @@ func (c *CommonCrawl) parseJSON(page string) []string {
 }
 
 func (c *CommonCrawl) getURL(domain, index string) string {
-	u, _ := url.Parse(c.baseURL + index + "-index")
+	u, _ := url.Parse(index)
 
 	u.RawQuery = url.Values{
 		"url":      {"*." + domain},
diff --git a/services/sources/crtsh.go b/services/sources/crtsh.go
index 3e93bc3e7..2250df3a3 100644
--- a/services/sources/crtsh.go
+++ b/services/sources/crtsh.go
@@ -5,6 +5,7 @@ package sources
 
 import (
 	"encoding/json"
+	"fmt"
 	"strings"
 
 	"github.com/OWASP/Amass/config"
@@ -44,7 +45,9 @@ func (c *Crtsh) OnStart() error {
 	var err error
 	c.db, err = sqlx.Connect("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable")
 	if err != nil {
-		c.Config().Log.Printf("%s: Failed to connect to the database server: %v", c.String(), err)
+		c.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Failed to connect to the database server: %v", c.String(), err),
+		)
 		c.haveConnection = false
 	}
 
@@ -86,7 +89,7 @@ func (c *Crtsh) executeQuery(domain string) {
 		WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1))
 		ORDER BY ci.NAME_VALUE`, pattern)
 	if err != nil {
-		c.Config().Log.Printf("%s: Query pattern %s: %v", c.String(), pattern, err)
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: Query pattern %s: %v", c.String(), pattern, err))
 		return
 	}
 
@@ -111,7 +114,7 @@ func (c *Crtsh) scrape(domain string) {
 	url := c.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		c.Config().Log.Printf("%s: %s: %v", c.String(), url, err)
+		c.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", c.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/dnsdb.go b/services/sources/dnsdb.go
index 851a5a446..3423c52f1 100644
--- a/services/sources/dnsdb.go
+++ b/services/sources/dnsdb.go
@@ -45,7 +45,7 @@ func (d *DNSDB) OnStart() error {
 
 	d.API = d.Config().GetAPIKey(d.String())
 	if d.API == nil || d.API.Key == "" {
-		d.Config().Log.Printf("%s: API key data was not provided", d.String())
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", d.String()))
 	}
 
 	go d.processRequests()
@@ -86,7 +86,7 @@ func (d *DNSDB) executeQuery(domain string) {
 		url := d.restURL(domain)
 		page, err := utils.RequestWebPage(url, nil, headers, "", "")
 		if err != nil {
-			d.Config().Log.Printf("%s: %s: %v", d.String(), url, err)
+			d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err))
 			return
 		}
 
@@ -142,7 +142,7 @@ func (d *DNSDB) scrape(domain string) {
 	url := d.getURL(domain, domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		d.Config().Log.Printf("%s: %s: %v", d.String(), url, err)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err))
 		return
 	}
 
@@ -177,7 +177,7 @@ loop:
 			url = d.getURL(domain, name)
 			another, err := utils.RequestWebPage(url, nil, nil, "", "")
 			if err != nil {
-				d.Config().Log.Printf("%s: %s: %v", d.String(), url, err)
+				d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err))
 				continue
 			}
 
diff --git a/services/sources/dnsdumpster.go b/services/sources/dnsdumpster.go
index 76cef078a..49622d62b 100644
--- a/services/sources/dnsdumpster.go
+++ b/services/sources/dnsdumpster.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
@@ -69,20 +70,20 @@ func (d *DNSDumpster) executeQuery(domain string) {
 	u := "https://dnsdumpster.com/"
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		d.Config().Log.Printf("%s: %s: %v", d.String(), u, err)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), u, err))
 		return
 	}
 
 	token := d.getCSRFToken(page)
 	if token == "" {
-		d.Config().Log.Printf("%s: %s: Failed to obtain the CSRF token", d.String(), u)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: Failed to obtain the CSRF token", d.String(), u))
 		return
 	}
 
 	d.SetActive()
 	page, err = d.postForm(token, domain)
 	if err != nil {
-		d.Config().Log.Printf("%s: %s: %v", d.String(), u, err)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), u, err))
 		return
 	}
 
@@ -120,7 +121,7 @@ func (d *DNSDumpster) postForm(token, domain string) (string, error) {
 
 	req, err := http.NewRequest("POST", "https://dnsdumpster.com/", strings.NewReader(params.Encode()))
 	if err != nil {
-		d.Config().Log.Printf("%s: Failed to setup the POST request: %v", d.String(), err)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: Failed to setup the POST request: %v", d.String(), err))
 		return "", err
 	}
 	// The CSRF token needs to be sent as a cookie
@@ -140,7 +141,7 @@ func (d *DNSDumpster) postForm(token, domain string) (string, error) {
 
 	resp, err := client.Do(req)
 	if err != nil {
-		d.Config().Log.Printf("%s: The POST request failed: %v", d.String(), err)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: The POST request failed: %v", d.String(), err))
 		return "", err
 	}
 	// Now, grab the entire page
diff --git a/services/sources/dnstable.go b/services/sources/dnstable.go
index 65e0cb91f..4d4b711d9 100644
--- a/services/sources/dnstable.go
+++ b/services/sources/dnstable.go
@@ -63,7 +63,7 @@ func (d *DNSTable) executeQuery(domain string) {
 	url := d.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		d.Config().Log.Printf("%s: %s: %v", d.String(), url, err)
+		d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/dogpile.go b/services/sources/dogpile.go
index 9681ceccf..388889443 100644
--- a/services/sources/dogpile.go
+++ b/services/sources/dogpile.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"time"
@@ -81,7 +82,7 @@ func (d *Dogpile) executeQuery(domain string) {
 			u := d.urlByPageNum(domain, i)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				d.Config().Log.Printf("%s: %s: %v", d.String(), u, err)
+				d.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", d.String(), u, err))
 				return
 			}
 
diff --git a/services/sources/entrust.go b/services/sources/entrust.go
index 004197d78..84d2de413 100644
--- a/services/sources/entrust.go
+++ b/services/sources/entrust.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"regexp"
 	"strings"
@@ -65,7 +66,7 @@ func (e *Entrust) executeQuery(domain string) {
 	u := e.getURL(domain)
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		e.Config().Log.Printf("%s: %s: %v", e.String(), u, err)
+		e.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", e.String(), u, err))
 		return
 	}
 	content := strings.Replace(page, "u003d", " ", -1)
diff --git a/services/sources/exalead.go b/services/sources/exalead.go
index 145fce826..5ac636ec7 100644
--- a/services/sources/exalead.go
+++ b/services/sources/exalead.go
@@ -63,7 +63,7 @@ func (e *Exalead) executeQuery(domain string) {
 	url := e.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		e.Config().Log.Printf("%s: %s: %v", e.String(), url, err)
+		e.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", e.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/google.go b/services/sources/google.go
index ffd07b1d2..60b7b12a4 100644
--- a/services/sources/google.go
+++ b/services/sources/google.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"time"
@@ -81,7 +82,7 @@ func (g *Google) executeQuery(domain string) {
 			u := g.urlByPageNum(domain, i)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				g.Config().Log.Printf("%s: %s: %v", g.String(), u, err)
+				g.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", g.String(), u, err))
 				return
 			}
 
diff --git a/services/sources/googlect.go b/services/sources/googlect.go
index 8e0e83391..0681d97ca 100644
--- a/services/sources/googlect.go
+++ b/services/sources/googlect.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"regexp"
 	"time"
@@ -81,7 +82,7 @@ func (g *GoogleCT) executeDNSQuery(domain string) {
 		}
 		page, err := utils.RequestWebPage(u, nil, headers, "", "")
 		if err != nil {
-			g.Config().Log.Printf("%s: %s: %v", g.String(), u, err)
+			g.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", g.String(), u, err))
 			break
 		}
 
diff --git a/services/sources/hackerone.go b/services/sources/hackerone.go
index f925ba59d..695949f60 100644
--- a/services/sources/hackerone.go
+++ b/services/sources/hackerone.go
@@ -64,7 +64,7 @@ func (h *HackerOne) executeDNSQuery(domain string) {
 	url := h.getDNSURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		h.Config().Log.Printf("%s: %s: %v", h.String(), url, err)
+		h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", h.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/hackertarget.go b/services/sources/hackertarget.go
index d11676e65..c3a3aa744 100644
--- a/services/sources/hackertarget.go
+++ b/services/sources/hackertarget.go
@@ -68,7 +68,7 @@ func (h *HackerTarget) executeDNSQuery(domain string) {
 	url := h.getDNSURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		h.Config().Log.Printf("%s: %s: %v", h.String(), url, err)
+		h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", h.String(), url, err))
 		return
 	}
 
@@ -96,19 +96,21 @@ func (h *HackerTarget) executeASNQuery(addr string) {
 	url := h.getASNURL(addr)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		h.Config().Log.Printf("%s: %s: %v", h.String(), url, err)
+		h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", h.String(), url, err))
 		return
 	}
 
 	fields := strings.Split(page, ",")
 	if len(fields) < 4 {
-		h.Config().Log.Printf("%s: %s: Failed to parse the response", h.String(), url)
+		h.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: Failed to parse the response", h.String(), url))
 		return
 	}
 
 	asn, err := strconv.Atoi(strings.Trim(fields[1], "\""))
 	if err != nil {
-		h.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", h.String(), url, err)
+		h.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", h.String(), url, err),
+		)
 		return
 	}
 
diff --git a/services/sources/ipv4info.go b/services/sources/ipv4info.go
index 665f3bed1..5754b7a0a 100644
--- a/services/sources/ipv4info.go
+++ b/services/sources/ipv4info.go
@@ -68,7 +68,7 @@ func (i *IPv4Info) executeQuery(domain string) {
 	url := i.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		i.Config().Log.Printf("%s: %s: %v", i.String(), url, err)
+		i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err))
 		return
 	}
 
@@ -77,7 +77,7 @@ func (i *IPv4Info) executeQuery(domain string) {
 	url = i.ipSubmatch(page, domain)
 	page, err = utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		i.Config().Log.Printf("%s: %s: %v", i.String(), url, err)
+		i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err))
 		return
 	}
 
@@ -86,7 +86,7 @@ func (i *IPv4Info) executeQuery(domain string) {
 	url = i.domainSubmatch(page, domain)
 	page, err = utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		i.Config().Log.Printf("%s: %s: %v", i.String(), url, err)
+		i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err))
 		return
 	}
 
@@ -95,7 +95,7 @@ func (i *IPv4Info) executeQuery(domain string) {
 	url = i.subdomainSubmatch(page, domain)
 	page, err = utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		i.Config().Log.Printf("%s: %s: %v", i.String(), url, err)
+		i.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", i.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/locarchive.go b/services/sources/locarchive.go
index da6983185..e959a597b 100644
--- a/services/sources/locarchive.go
+++ b/services/sources/locarchive.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -70,7 +72,7 @@ func (l *LoCArchive) executeQuery(sn, domain string) {
 
 	names, err := crawl(l, l.baseURL, l.domain, sn, domain)
 	if err != nil {
-		l.Config().Log.Printf("%s: %v", l.String(), err)
+		l.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", l.String(), err))
 		return
 	}
 
diff --git a/services/sources/mnemonic.go b/services/sources/mnemonic.go
index 3b440cc06..8be2e9e37 100644
--- a/services/sources/mnemonic.go
+++ b/services/sources/mnemonic.go
@@ -61,7 +61,7 @@ func (m *Mnemonic) executeDNSQuery(domain string) {
 	url := m.getDNSURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		m.Config().Log.Printf("%s: %s: %v", m.String(), url, err)
+		m.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", m.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/netcraft.go b/services/sources/netcraft.go
index 4229c946c..df65ce2ee 100644
--- a/services/sources/netcraft.go
+++ b/services/sources/netcraft.go
@@ -63,7 +63,7 @@ func (n *Netcraft) executeQuery(domain string) {
 	url := n.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s, %v", n.String(), url, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/networksdb.go b/services/sources/networksdb.go
index 57ea41f69..30a191e4d 100644
--- a/services/sources/networksdb.go
+++ b/services/sources/networksdb.go
@@ -5,6 +5,7 @@ package sources
 
 import (
 	"encoding/json"
+	"fmt"
 	"net"
 	"net/url"
 	"regexp"
@@ -62,7 +63,7 @@ func (n *NetworksDB) OnStart() error {
 
 	n.API = n.Config().GetAPIKey(n.String())
 	if n.API == nil || n.API.Key == "" {
-		n.Config().Log.Printf("%s: API key data was not provided", n.String())
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", n.String()))
 		n.SourceType = requests.SCRAPE
 		n.hasAPIKey = false
 	}
@@ -113,13 +114,15 @@ func (n *NetworksDB) executeASNAddrQuery(addr string) {
 	u := n.getIPURL(addr)
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return
 	}
 
 	matches := networksdbOrgLinkRE.FindStringSubmatch(page)
 	if matches == nil || len(matches) < 2 {
-		n.Config().Log.Printf("%s: %s: Failed to extract the organization info href", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to extract the organization info href", n.String(), u),
+		)
 		return
 	}
 
@@ -128,7 +131,7 @@ func (n *NetworksDB) executeASNAddrQuery(addr string) {
 	u = networksdbBaseURL + matches[1]
 	page, err = utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return
 	}
 
@@ -141,13 +144,17 @@ func (n *NetworksDB) executeASNAddrQuery(addr string) {
 
 	matches = networksdbASNRE.FindStringSubmatch(page)
 	if matches == nil || len(matches) < 2 {
-		n.Config().Log.Printf("%s: %s: The regular expression failed to extract the ASN", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The regular expression failed to extract the ASN", n.String(), u),
+		)
 		return
 	}
 
 	asn, err := strconv.Atoi(strings.TrimSpace(matches[1]))
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: Failed to extract a valid ASN", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to extract a valid ASN", n.String(), u),
+		)
 		return
 	}
 
@@ -165,20 +172,24 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) {
 	u := n.getASNURL(asn)
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return
 	}
 
 	matches := networksdbASNameRE.FindStringSubmatch(page)
 	if matches == nil || len(matches) < 2 {
-		n.Config().Log.Printf("%s: The regular expression failed to extract the AS name", n.String())
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: The regular expression failed to extract the AS name", n.String()),
+		)
 		return
 	}
 	name := strings.TrimSpace(matches[1])
 
 	matches = networksdbCCRE.FindStringSubmatch(page)
 	if matches == nil || len(matches) < 2 {
-		n.Config().Log.Printf("%s: The regular expression failed to extract the country code", n.String())
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: The regular expression failed to extract the country code", n.String()),
+		)
 		return
 	}
 	cc := strings.TrimSpace(matches[1])
@@ -223,14 +234,18 @@ func (n *NetworksDB) getASNURL(asn int) string {
 func (n *NetworksDB) executeAPIASNAddrQuery(addr string) {
 	_, id := n.apiIPQuery(addr)
 	if id == "" {
-		n.Config().Log.Printf("%s: %s: Failed to obtain IP address information", n.String(), addr)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to obtain IP address information", n.String(), addr),
+		)
 		return
 	}
 
 	time.Sleep(n.RateLimit)
 	asns := n.apiOrgInfoQuery(id)
 	if len(asns) == 0 {
-		n.Config().Log.Printf("%s: %s: Failed to obtain ASNs associated with the organization", n.String(), id)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to obtain ASNs associated with the organization", n.String(), id),
+		)
 		return
 	}
 
@@ -242,7 +257,9 @@ loop:
 		time.Sleep(n.RateLimit)
 		cidrs = n.apiNetblocksQuery(a)
 		if len(cidrs) == 0 {
-			n.Config().Log.Printf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), a)
+			n.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), a),
+			)
 		}
 
 		for _, cidr := range cidrs {
@@ -256,7 +273,9 @@ loop:
 	}
 
 	if asn == 0 {
-		n.Config().Log.Printf("%s: %s: Failed to obtain the ASN associated with the IP address", n.String(), addr)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to obtain the ASN associated with the IP address", n.String(), addr),
+		)
 		return
 	}
 	n.executeAPIASNQuery(asn, addr, cidrs)
@@ -266,7 +285,9 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string
 	if netblocks == nil {
 		netblocks = n.apiNetblocksQuery(asn)
 		if len(netblocks) == 0 {
-			n.Config().Log.Printf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), asn)
+			n.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), asn),
+			)
 			return
 		}
 	}
@@ -288,7 +309,9 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string
 	time.Sleep(n.RateLimit)
 	req := n.apiASNInfoQuery(asn)
 	if req == nil {
-		n.Config().Log.Printf("%s: %d: Failed to obtain ASN information", n.String(), asn)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %d: Failed to obtain ASN information", n.String(), asn),
+		)
 		return
 	}
 
@@ -307,7 +330,7 @@ func (n *NetworksDB) apiIPQuery(addr string) (string, string) {
 	body := strings.NewReader(params.Encode())
 	page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return "", ""
 	}
 
@@ -324,13 +347,15 @@ func (n *NetworksDB) apiIPQuery(addr string) (string, string) {
 		} `json:"results"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return "", ""
 	} else if m.Error != "" {
-		n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error))
 		return "", ""
 	} else if m.Total == 0 || len(m.Results) == 0 {
-		n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u),
+		)
 		return "", ""
 	}
 
@@ -348,7 +373,7 @@ func (n *NetworksDB) apiOrgInfoQuery(id string) []int {
 	body := strings.NewReader(params.Encode())
 	page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return []int{}
 	}
 
@@ -360,13 +385,15 @@ func (n *NetworksDB) apiOrgInfoQuery(id string) []int {
 		} `json:"results"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return []int{}
 	} else if m.Error != "" {
-		n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error))
 		return []int{}
 	} else if m.Total == 0 || len(m.Results[0].ASNs) == 0 {
-		n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u),
+		)
 		return []int{}
 	}
 
@@ -384,7 +411,7 @@ func (n *NetworksDB) apiASNInfoQuery(asn int) *requests.ASNRequest {
 	body := strings.NewReader(params.Encode())
 	page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return nil
 	}
 
@@ -400,13 +427,15 @@ func (n *NetworksDB) apiASNInfoQuery(asn int) *requests.ASNRequest {
 		} `json:"results"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return nil
 	} else if m.Error != "" {
-		n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error))
 		return nil
 	} else if m.Total == 0 || len(m.Results) == 0 {
-		n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u),
+		)
 		return nil
 	}
 
@@ -432,7 +461,7 @@ func (n *NetworksDB) apiNetblocksQuery(asn int) []string {
 	body := strings.NewReader(params.Encode())
 	page, err := utils.RequestWebPage(u, body, n.getHeaders(), "", "")
 	if err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return netblocks
 	}
 
@@ -444,13 +473,15 @@ func (n *NetworksDB) apiNetblocksQuery(asn int) []string {
 		} `json:"results"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		n.Config().Log.Printf("%s: %s: %v", n.String(), u, err)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", n.String(), u, err))
 		return netblocks
 	} else if m.Error != "" {
-		n.Config().Log.Printf("%s: %s: %s", n.String(), u, m.Error)
+		n.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %s", n.String(), u, m.Error))
 		return netblocks
 	} else if m.Total == 0 || len(m.Results) == 0 {
-		n.Config().Log.Printf("%s: %s: The request returned zero results", n.String(), u)
+		n.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The request returned zero results", n.String(), u),
+		)
 		return netblocks
 	}
 
diff --git a/services/sources/openukarchive.go b/services/sources/openukarchive.go
index 25c312fb4..ed8fb9496 100644
--- a/services/sources/openukarchive.go
+++ b/services/sources/openukarchive.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -67,7 +69,7 @@ func (o *OpenUKArchive) executeQuery(sn, domain string) {
 
 	names, err := crawl(o, o.baseURL, o.domain, sn, domain)
 	if err != nil {
-		o.Config().Log.Printf("%s: %v", o.String(), err)
+		o.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", o.String(), err))
 		return
 	}
 
diff --git a/services/sources/passivetotal.go b/services/sources/passivetotal.go
index 8f5632a5c..03f36d48c 100644
--- a/services/sources/passivetotal.go
+++ b/services/sources/passivetotal.go
@@ -5,6 +5,7 @@ package sources
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/OWASP/Amass/config"
@@ -41,7 +42,7 @@ func (pt *PassiveTotal) OnStart() error {
 
 	pt.API = pt.Config().GetAPIKey(pt.String())
 	if pt.API == nil || pt.API.Username == "" || pt.API.Key == "" {
-		pt.Config().Log.Printf("%s: API key data was not provided", pt.String())
+		pt.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: API key data was not provided", pt.String()))
 	}
 
 	go pt.processRequests()
@@ -86,7 +87,7 @@ func (pt *PassiveTotal) executeQuery(domain string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, pt.API.Username, pt.API.Key)
 	if err != nil {
-		pt.Config().Log.Printf("%s: %s: %v", pt.String(), url, err)
+		pt.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", pt.String(), url, err))
 		return
 	}
 	// Extract the subdomain names from the REST API results
diff --git a/services/sources/ptrarchive.go b/services/sources/ptrarchive.go
index 32a99b3c5..a73e53ccf 100644
--- a/services/sources/ptrarchive.go
+++ b/services/sources/ptrarchive.go
@@ -63,7 +63,7 @@ func (p *PTRArchive) executeQuery(domain string) {
 	url := p.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		p.Config().Log.Printf("%s: %s: %v", p.String(), url, err)
+		p.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", p.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/radb.go b/services/sources/radb.go
index 5c3089406..3b91c735e 100644
--- a/services/sources/radb.go
+++ b/services/sources/radb.go
@@ -120,7 +120,7 @@ func (r *RADb) executeASNAddrQuery(addr string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return
 	}
 
@@ -134,10 +134,12 @@ func (r *RADb) executeASNAddrQuery(addr string) {
 		} `json:"cidr0_cidrs"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return
 	} else if m.ClassName != "ip network" || len(m.CIDRs) == 0 {
-		r.Config().Log.Printf("%s: %s: The request returned zero results", r.String(), url)
+		r.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The request returned zero results", r.String(), url),
+		)
 		return
 	}
 
@@ -176,7 +178,7 @@ func (r *RADb) executeASNQuery(asn int, prefix string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return
 	}
 
@@ -189,10 +191,12 @@ func (r *RADb) executeASNQuery(asn int, prefix string) {
 		}
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return
 	} else if m.ClassName != "autnum" {
-		r.Config().Log.Printf("%s: %s: The query returned incorrect results", r.String(), url)
+		r.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The query returned incorrect results", r.String(), url),
+		)
 		return
 	}
 
@@ -222,7 +226,9 @@ func (r *RADb) executeASNQuery(asn int, prefix string) {
 	blocks = utils.UniqueAppend(blocks, r.netblocks(asn)...)
 
 	if len(blocks) == 0 {
-		r.Config().Log.Printf("%s: %s: The query returned zero netblocks", r.String(), url)
+		r.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The query returned zero netblocks", r.String(), url),
+		)
 		return
 	}
 
@@ -251,7 +257,7 @@ func (r *RADb) netblocks(asn int) []string {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return netblocks
 	}
 
@@ -267,7 +273,7 @@ func (r *RADb) netblocks(asn int) []string {
 		} `json:"arin_originas0_networkSearchResults"`
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return netblocks
 	}
 
@@ -295,7 +301,9 @@ func (r *RADb) netblocks(asn int) []string {
 	}
 
 	if len(netblocks) == 0 {
-		r.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn)
+		r.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn),
+		)
 	}
 	return netblocks
 }
@@ -311,13 +319,15 @@ func (r *RADb) ipToASN(cidr string) int {
 	if r.addr == "" {
 		answers, err := r.Pool().Resolve(radbWhoisURL, "A", resolvers.PriorityHigh)
 		if err != nil {
-			r.Config().Log.Printf("%s: %s: %v", r.String(), radbWhoisURL, err)
+			r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), radbWhoisURL, err))
 			return 0
 		}
 
 		ip := answers[0].Data
 		if ip == "" {
-			r.Config().Log.Printf("%s: Failed to resolve %s", r.String(), radbWhoisURL)
+			r.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: Failed to resolve %s", r.String(), radbWhoisURL),
+			)
 			return 0
 		}
 		r.addr = ip
@@ -329,7 +339,7 @@ func (r *RADb) ipToASN(cidr string) int {
 	d := net.Dialer{}
 	conn, err := d.DialContext(ctx, "tcp", r.addr+":43")
 	if err != nil {
-		r.Config().Log.Printf("%s: %v", r.String(), err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", r.String(), err))
 		return 0
 	}
 	defer conn.Close()
diff --git a/services/sources/riddler.go b/services/sources/riddler.go
index b417db509..0e8cd1284 100644
--- a/services/sources/riddler.go
+++ b/services/sources/riddler.go
@@ -63,7 +63,7 @@ func (r *Riddler) executeQuery(domain string) {
 	url := r.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/robtex.go b/services/sources/robtex.go
index 7ee43f3a6..8bee33a70 100644
--- a/services/sources/robtex.go
+++ b/services/sources/robtex.go
@@ -6,6 +6,7 @@ package sources
 import (
 	"bufio"
 	"encoding/json"
+	"fmt"
 	"net"
 	"strconv"
 	"strings"
@@ -101,7 +102,7 @@ func (r *Robtex) executeDNSQuery(domain string) {
 	url := "https://freeapi.robtex.com/pdns/forward/" + domain
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return
 	}
 
@@ -132,7 +133,7 @@ loop:
 			url = "https://freeapi.robtex.com/pdns/reverse/" + ip
 			pdns, err := utils.RequestWebPage(url, nil, nil, "", "")
 			if err != nil {
-				r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+				r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 				continue
 			}
 
@@ -220,7 +221,7 @@ func (r *Robtex) origin(addr string) *requests.ASNRequest {
 	url := "https://freeapi.robtex.com/ipquery/" + addr
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return nil
 	}
 	// Extract the network information
@@ -293,7 +294,9 @@ func (r *Robtex) origin(addr string) *requests.ASNRequest {
 	}
 
 	if ipinfo.ASN == 0 {
-		r.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", r.String(), url, err),
+		)
 		return nil
 	}
 
@@ -322,7 +325,7 @@ func (r *Robtex) netblocks(asn int) []string {
 	url := "https://freeapi.robtex.com/asquery/" + strconv.Itoa(asn)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		r.Config().Log.Printf("%s: %s: %v", r.String(), url, err)
+		r.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", r.String(), url, err))
 		return netblocks
 	}
 	// Extract the network information
@@ -341,7 +344,9 @@ func (r *Robtex) netblocks(asn int) []string {
 	}
 
 	if len(netblocks) == 0 {
-		r.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn)
+		r.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Failed to acquire netblocks for ASN %d", r.String(), asn),
+		)
 	}
 	return netblocks
 }
diff --git a/services/sources/securitytrails.go b/services/sources/securitytrails.go
index 5ad64733a..ed67f4dfc 100644
--- a/services/sources/securitytrails.go
+++ b/services/sources/securitytrails.go
@@ -43,7 +43,9 @@ func (st *SecurityTrails) OnStart() error {
 
 	st.API = st.Config().GetAPIKey(st.String())
 	if st.API == nil || st.API.Key == "" {
-		st.Config().Log.Printf("%s: API key data was not provided", st.String())
+		st.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", st.String()),
+		)
 	}
 
 	go st.processRequests()
@@ -88,7 +90,7 @@ func (st *SecurityTrails) executeQuery(domain string) {
 	st.SetActive()
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		st.Config().Log.Printf("%s: %s: %v", st.String(), url, err)
+		st.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", st.String(), url, err))
 		return
 	}
 	// Extract the subdomain names from the REST API results
diff --git a/services/sources/shadowserver.go b/services/sources/shadowserver.go
index d340af5bd..f206e3f34 100644
--- a/services/sources/shadowserver.go
+++ b/services/sources/shadowserver.go
@@ -127,19 +127,25 @@ func (s *ShadowServer) origin(addr string) *requests.ASNRequest {
 
 	answers, err := s.Pool().Resolve(name, "TXT", resolvers.PriorityHigh)
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: DNS TXT record query error: %v", s.String(), name, err)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: DNS TXT record query error: %v", s.String(), name, err),
+		)
 		return nil
 	}
 
 	fields := strings.Split(strings.Trim(answers[0].Data, "\""), " | ")
 	if len(fields) < 5 {
-		s.Config().Log.Printf("%s: %s: Failed to parse the origin response", s.String(), name)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response", s.String(), name),
+		)
 		return nil
 	}
 
 	asn, err := strconv.Atoi(strings.TrimSpace(fields[0]))
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", s.String(), name, err)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", s.String(), name, err),
+		)
 		return nil
 	}
 
@@ -160,13 +166,17 @@ func (s *ShadowServer) netblocks(asn int) []string {
 	if s.addr == "" {
 		answers, err := s.Pool().Resolve(ShadowServerWhoisURL, "A", resolvers.PriorityHigh)
 		if err != nil {
-			s.Config().Log.Printf("%s: %s: %v", s.String(), ShadowServerWhoisURL, err)
+			s.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: %s: %v", s.String(), ShadowServerWhoisURL, err),
+			)
 			return netblocks
 		}
 
 		ip := answers[0].Data
 		if ip == "" {
-			s.Config().Log.Printf("%s: Failed to resolve %s", s.String(), ShadowServerWhoisURL)
+			s.Bus().Publish(requests.LogTopic,
+				fmt.Sprintf("%s: Failed to resolve %s", s.String(), ShadowServerWhoisURL),
+			)
 			return netblocks
 		}
 		s.addr = ip
@@ -178,7 +188,7 @@ func (s *ShadowServer) netblocks(asn int) []string {
 	d := net.Dialer{}
 	conn, err := d.DialContext(ctx, "tcp", s.addr+":43")
 	if err != nil {
-		s.Config().Log.Printf("%s: %v", s.String(), err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", s.String(), err))
 		return netblocks
 	}
 	defer conn.Close()
@@ -195,7 +205,9 @@ func (s *ShadowServer) netblocks(asn int) []string {
 	}
 
 	if len(netblocks) == 0 {
-		s.Config().Log.Printf("%s: Failed to acquire netblocks for ASN %d", s.String(), asn)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Failed to acquire netblocks for ASN %d", s.String(), asn),
+		)
 	}
 	return netblocks
 }
diff --git a/services/sources/shodan.go b/services/sources/shodan.go
index 3f39a696a..70d17dc30 100644
--- a/services/sources/shodan.go
+++ b/services/sources/shodan.go
@@ -42,7 +42,9 @@ func (s *Shodan) OnStart() error {
 
 	s.API = s.Config().GetAPIKey(s.String())
 	if s.API == nil || s.API.Key == "" {
-		s.Config().Log.Printf("%s: API key data was not provided", s.String())
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", s.String()),
+		)
 	}
 
 	go s.processRequests()
@@ -83,7 +85,7 @@ func (s *Shodan) executeQuery(domain string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), url, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err))
 		return
 	}
 	// Extract the subdomain names from the REST API results
diff --git a/services/sources/sitedossier.go b/services/sources/sitedossier.go
index 8beab497d..32e1aa803 100644
--- a/services/sources/sitedossier.go
+++ b/services/sources/sitedossier.go
@@ -63,7 +63,7 @@ func (s *SiteDossier) executeQuery(domain string) {
 	url := s.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), url, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/spyse.go b/services/sources/spyse.go
index a8e997e6a..1e6517ce2 100644
--- a/services/sources/spyse.go
+++ b/services/sources/spyse.go
@@ -37,7 +37,9 @@ func (s *Spyse) OnStart() error {
 
 	s.API = s.Config().GetAPIKey(s.String())
 	if s.API == nil || s.API.Key == "" {
-		s.Config().Log.Printf("%s: API key data was not provided", s.String())
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", s.String()),
+		)
 	}
 
 	go s.processRequests()
@@ -83,7 +85,7 @@ func (s *Spyse) subdomainQueryAPI(domain string, page int) (int, error) {
 	u := s.getAPIURL(domain, page)
 	response, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), u, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), u, err))
 		return 0, err
 	}
 
@@ -95,7 +97,9 @@ func (s *Spyse) subdomainQueryAPI(domain string, page int) (int, error) {
 	}
 
 	if err := json.Unmarshal([]byte(response), &results); err != nil {
-		s.Config().Log.Printf("%s: Failed to unmarshal JSON: %v", s.String(), err)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Failed to unmarshal JSON: %v", s.String(), err),
+		)
 		return 0, err
 	}
 
@@ -128,7 +132,7 @@ func (s *Spyse) executeSubdomainQuery(domain string) {
 	url := s.getURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), url, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err))
 		return
 	}
 
@@ -148,7 +152,7 @@ func (s *Spyse) certQueryAPI(domain string) error {
 	u := s.getCertAPIURL(domain)
 	response, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), u, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), u, err))
 		return err
 	}
 
@@ -159,7 +163,9 @@ func (s *Spyse) certQueryAPI(domain string) error {
 	}
 
 	if err := json.Unmarshal([]byte(response), &results); err != nil {
-		s.Config().Log.Printf("%s: Failed to unmarshal JSON: %v", s.String(), err)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: Failed to unmarshal JSON: %v", s.String(), err),
+		)
 		return err
 	}
 
diff --git a/services/sources/sublist3r.go b/services/sources/sublist3r.go
index 46665ad16..e731fe6c4 100644
--- a/services/sources/sublist3r.go
+++ b/services/sources/sublist3r.go
@@ -5,6 +5,7 @@ package sources
 
 import (
 	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/OWASP/Amass/config"
@@ -19,7 +20,6 @@ import (
 type Sublist3rAPI struct {
 	services.BaseService
 
-	API        *config.APIKey
 	SourceType string
 	RateLimit  time.Duration
 }
@@ -71,17 +71,19 @@ func (s *Sublist3rAPI) executeQuery(domain string) {
 	url := s.restURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), url, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err))
 		return
 	}
 
 	// Extract the subdomain names from the REST API results
 	var subs []string
 	if err := json.Unmarshal([]byte(page), &subs); err != nil {
-		s.Config().Log.Printf("%s: %s: %v", s.String(), url, err)
+		s.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", s.String(), url, err))
 		return
 	} else if len(subs) == 0 {
-		s.Config().Log.Printf("%s: %s: The request returned zero results", s.String(), url)
+		s.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: The request returned zero results", s.String(), url),
+		)
 		return
 	}
 
diff --git a/services/sources/teamcymru.go b/services/sources/teamcymru.go
index f432df1c8..55b5f2f8a 100644
--- a/services/sources/teamcymru.go
+++ b/services/sources/teamcymru.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net"
 	"strconv"
 	"strings"
@@ -96,25 +97,33 @@ func (t *TeamCymru) origin(addr string) *requests.ASNRequest {
 	} else if utils.IsIPv6(ip) {
 		name = utils.IPv6NibbleFormat(utils.HexString(ip)) + ".origin6.asn.cymru.com"
 	} else {
-		t.Config().Log.Printf("%s: %s: Failed to parse the IP address", t.String(), addr)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the IP address", t.String(), addr),
+		)
 		return nil
 	}
 
 	answers, err = t.Pool().Resolve(name, "TXT", resolvers.PriorityHigh)
 	if err != nil {
-		t.Config().Log.Printf("%s: %s: DNS TXT record query error: %v", t.String(), name, err)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: DNS TXT record query error: %v", t.String(), name, err),
+		)
 		return nil
 	}
 
 	fields := strings.Split(answers[0].Data, " | ")
 	if len(fields) < 5 {
-		t.Config().Log.Printf("%s: %s: Failed to parse the origin response", t.String(), name)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response", t.String(), name),
+		)
 		return nil
 	}
 
 	asn, err := strconv.Atoi(strings.TrimSpace(fields[0]))
 	if err != nil {
-		t.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err),
+		)
 		return nil
 	}
 
@@ -142,19 +151,25 @@ func (t *TeamCymru) asnLookup(asn int) *requests.ASNRequest {
 
 	answers, err = t.Pool().Resolve(name, "TXT", resolvers.PriorityHigh)
 	if err != nil {
-		t.Config().Log.Printf("%s: %s: DNS TXT record query error: %v", t.String(), name, err)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: DNS TXT record query error: %v", t.String(), name, err),
+		)
 		return nil
 	}
 
 	fields := strings.Split(answers[0].Data, " | ")
 	if len(fields) < 5 {
-		t.Config().Log.Printf("%s: %s: Failed to parse the origin response", t.String(), name)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response", t.String(), name),
+		)
 		return nil
 	}
 
 	pASN, err := strconv.Atoi(strings.TrimSpace(fields[0]))
 	if err != nil || asn != pASN {
-		t.Config().Log.Printf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to parse the origin response: %v", t.String(), name, err),
+		)
 		return nil
 	}
 
diff --git a/services/sources/threatcrowd.go b/services/sources/threatcrowd.go
index a7c268de0..2b932aba7 100644
--- a/services/sources/threatcrowd.go
+++ b/services/sources/threatcrowd.go
@@ -78,7 +78,7 @@ func (t *ThreatCrowd) executeQuery(domain string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		t.Config().Log.Printf("%s: %s: %v", t.String(), url, err)
+		t.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", t.String(), url, err))
 		return
 	}
 
@@ -95,7 +95,9 @@ func (t *ThreatCrowd) executeQuery(domain string) {
 	}
 
 	if m.ResponseCode != "1" {
-		t.Config().Log.Printf("%s: %s: Response code %s", t.String(), url, m.ResponseCode)
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Response code %s", t.String(), url, m.ResponseCode),
+		)
 		return
 	}
 
diff --git a/services/sources/twitter.go b/services/sources/twitter.go
index f95ae6f74..5ffdf9f41 100644
--- a/services/sources/twitter.go
+++ b/services/sources/twitter.go
@@ -46,7 +46,9 @@ func (t *Twitter) OnStart() error {
 
 	t.API = t.Config().GetAPIKey(t.String())
 	if t.API == nil || t.API.Key == "" || t.API.Secret == "" {
-		t.Config().Log.Printf("%s: API key data was not provided", t.String())
+		t.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", t.String()),
+		)
 	}
 	if t.API != nil && t.API.Key != "" && t.API.Secret != "" {
 		if bearer, err := t.getBearerToken(); err == nil {
@@ -99,7 +101,7 @@ func (t *Twitter) executeQuery(domain string) {
 	t.SetActive()
 	search, _, err := t.client.Search.Tweets(searchParams)
 	if err != nil {
-		t.Config().Log.Printf("%s: %v", t.String(), err)
+		t.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", t.String(), err))
 		return
 	}
 
diff --git a/services/sources/ukgovarchive.go b/services/sources/ukgovarchive.go
index 91a4e4a5b..0ff74cf55 100644
--- a/services/sources/ukgovarchive.go
+++ b/services/sources/ukgovarchive.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -23,7 +25,7 @@ type UKGovArchive struct {
 }
 
 // NewUKGovArchive returns he object initialized, but not yet started.
-func NewUKGovArchive(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *UKGovArchive {
+func NewUKGovArchive(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *UKGovArchive {
 	u := &UKGovArchive{
 		domain:     "webarchive.nationalarchives.gov.uk",
 		baseURL:    "http://webarchive.nationalarchives.gov.uk",
@@ -31,7 +33,7 @@ func NewUKGovArchive(c *config.Config, bus *eb.EventBus, pool *resolvers.Resolve
 		filter:     utils.NewStringFilter(),
 	}
 
-	u.BaseService = *services.NewBaseService(u, "UKGovArchive", c, bus, pool)
+	u.BaseService = *services.NewBaseService(u, "UKGovArchive", cfg, bus, pool)
 	return u
 }
 
@@ -67,7 +69,7 @@ func (u *UKGovArchive) executeQuery(sn, domain string) {
 
 	names, err := crawl(u, u.baseURL, u.domain, sn, domain)
 	if err != nil {
-		u.Config().Log.Printf("%s: %v", u.String(), err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", u.String(), err))
 		return
 	}
 
diff --git a/services/sources/umbrella.go b/services/sources/umbrella.go
index d59ce929c..4cb76db72 100644
--- a/services/sources/umbrella.go
+++ b/services/sources/umbrella.go
@@ -43,7 +43,9 @@ func (u *Umbrella) OnStart() error {
 
 	u.API = u.Config().GetAPIKey(u.String())
 	if u.API == nil || u.API.Key == "" {
-		u.Config().Log.Printf("%s: API key data was not provided", u.String())
+		u.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", u.String()),
+		)
 	}
 
 	go u.processRequests()
@@ -92,7 +94,7 @@ func (u *Umbrella) executeDNSQuery(domain string) {
 	url := u.patternSearchRestURL(domain)
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err))
 		return
 	}
 
@@ -108,7 +110,7 @@ func (u *Umbrella) executeDNSQuery(domain string) {
 	url = u.occurrencesRestURL(domain)
 	page, err = utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err))
 		return
 	}
 
@@ -128,7 +130,7 @@ func (u *Umbrella) executeDNSQuery(domain string) {
 	url = u.relatedRestURL(domain)
 	page, err = utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err))
 		return
 	}
 
@@ -199,13 +201,13 @@ func (u *Umbrella) queryWhois(domain string) *whoisRecord {
 	u.SetActive()
 	record, err := utils.RequestWebPage(whoisURL, nil, headers, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), whoisURL, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), whoisURL, err))
 		return nil
 	}
 
 	err = json.Unmarshal([]byte(record), &whois)
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), whoisURL, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), whoisURL, err))
 		return nil
 	}
 
@@ -225,7 +227,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string {
 		fullAPIURL := fmt.Sprintf("%s&offset=%d", apiURL, count)
 		record, err := utils.RequestWebPage(fullAPIURL, nil, headers, "", "")
 		if err != nil {
-			u.Config().Log.Printf("%s: %s: %v", u.String(), apiURL, err)
+			u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), apiURL, err))
 			return domains
 		}
 		err = json.Unmarshal([]byte(record), &whois)
diff --git a/services/sources/urlscan.go b/services/sources/urlscan.go
index 9753e3cee..d3204f476 100644
--- a/services/sources/urlscan.go
+++ b/services/sources/urlscan.go
@@ -43,7 +43,9 @@ func (u *URLScan) OnStart() error {
 
 	u.API = u.Config().GetAPIKey(u.String())
 	if u.API == nil || u.API.Key == "" {
-		u.Config().Log.Printf("%s: API key data was not provided", u.String())
+		u.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", u.String()),
+		)
 	}
 
 	go u.processRequests()
@@ -83,7 +85,7 @@ func (u *URLScan) executeQuery(domain string) {
 	url := u.searchURL(domain)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err))
 		return
 	}
 	// Extract the subdomain names from the REST API results
@@ -131,7 +133,7 @@ func (u *URLScan) getSubsFromResult(id string) []string {
 	url := u.resultURL(id)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err))
 		return subs
 	}
 	// Extract the subdomain names from the REST API results
@@ -160,7 +162,7 @@ func (u *URLScan) attemptSubmission(domain string) string {
 	body := strings.NewReader(u.submitBody(domain))
 	page, err := utils.RequestWebPage(url, body, headers, "", "")
 	if err != nil {
-		u.Config().Log.Printf("%s: %s: %v", u.String(), url, err)
+		u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), url, err))
 		return ""
 	}
 	// Extract the subdomain names from the REST API results
diff --git a/services/sources/viewdns.go b/services/sources/viewdns.go
index 77bd2da48..d7c25385c 100644
--- a/services/sources/viewdns.go
+++ b/services/sources/viewdns.go
@@ -83,7 +83,7 @@ func (v *ViewDNS) executeDNSQuery(domain string) {
 	// The ViewDNS IP History lookup sometimes reveals interesting results
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		v.Config().Log.Printf("%s: %s: %v", v.String(), u, err)
+		v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), u, err))
 		return
 	}
 
@@ -107,13 +107,15 @@ func (v *ViewDNS) executeWhoisQuery(domain string) {
 	u := v.getURL(domain)
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
-		v.Config().Log.Printf("%s: %s: %v", v.String(), u, err)
+		v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), u, err))
 		return
 	}
 	// Pull the table we need from the page content
 	table := getViewDNSTable(page)
 	if table == "" {
-		v.Config().Log.Printf("%s: %s: Failed to discover the table of results", v.String(), u)
+		v.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Failed to discover the table of results", v.String(), u),
+		)
 		return
 	}
 	// Get the list of domain names discovered through the reverse DNS service
diff --git a/services/sources/virustotal.go b/services/sources/virustotal.go
index 4243c33d6..82f78f1a8 100644
--- a/services/sources/virustotal.go
+++ b/services/sources/virustotal.go
@@ -30,14 +30,14 @@ type VirusTotal struct {
 }
 
 // NewVirusTotal returns he object initialized, but not yet started.
-func NewVirusTotal(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *VirusTotal {
+func NewVirusTotal(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *VirusTotal {
 	v := &VirusTotal{
 		SourceType: requests.API,
 		RateLimit:  15 * time.Second,
 		haveAPIKey: true,
 	}
 
-	v.BaseService = *services.NewBaseService(v, "VirusTotal", c, bus, pool)
+	v.BaseService = *services.NewBaseService(v, "VirusTotal", cfg, bus, pool)
 	return v
 }
 
@@ -48,7 +48,9 @@ func (v *VirusTotal) OnStart() error {
 	v.API = v.Config().GetAPIKey(v.String())
 	if v.API == nil || v.API.Key == "" {
 		v.haveAPIKey = false
-		v.Config().Log.Printf("%s: API key data was not provided", v.String())
+		v.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: API key data was not provided", v.String()),
+		)
 	}
 
 	go v.processRequests()
@@ -93,7 +95,7 @@ func (v *VirusTotal) apiQuery(domain string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		v.Config().Log.Printf("%s: %s: %v", v.String(), url, err)
+		v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), url, err))
 		return
 	}
 
@@ -111,7 +113,9 @@ func (v *VirusTotal) apiQuery(domain string) {
 	}
 
 	if m.ResponseCode != 1 {
-		v.Config().Log.Printf("%s: %s: Response code %d: %s", v.String(), url, m.ResponseCode, m.Message)
+		v.Bus().Publish(requests.LogTopic,
+			fmt.Sprintf("%s: %s: Response code %d: %s", v.String(), url, m.ResponseCode, m.Message),
+		)
 		return
 	}
 
@@ -156,7 +160,7 @@ func (v *VirusTotal) regularQuery(domain string) {
 	headers := map[string]string{"Content-Type": "application/json"}
 	page, err := utils.RequestWebPage(url, nil, headers, "", "")
 	if err != nil {
-		v.Config().Log.Printf("%s: %s: %v", v.String(), url, err)
+		v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), url, err))
 		return
 	}
 
diff --git a/services/sources/wayback.go b/services/sources/wayback.go
index 0468a80d5..00cb5eb92 100644
--- a/services/sources/wayback.go
+++ b/services/sources/wayback.go
@@ -4,6 +4,8 @@
 package sources
 
 import (
+	"fmt"
+
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
@@ -16,22 +18,22 @@ import (
 type Wayback struct {
 	services.BaseService
 
+	SourceType string
 	domain     string
 	baseURL    string
-	SourceType string
 	filter     *utils.StringFilter
 }
 
 // NewWayback returns he object initialized, but not yet started.
-func NewWayback(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Wayback {
+func NewWayback(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Wayback {
 	w := &Wayback{
+		SourceType: requests.ARCHIVE,
 		domain:     "web.archive.org",
 		baseURL:    "http://web.archive.org/web",
-		SourceType: requests.ARCHIVE,
 		filter:     utils.NewStringFilter(),
 	}
 
-	w.BaseService = *services.NewBaseService(w, "Wayback", c, bus, pool)
+	w.BaseService = *services.NewBaseService(w, "Wayback", cfg, bus, pool)
 	return w
 }
 
@@ -67,7 +69,7 @@ func (w *Wayback) executeQuery(sn, domain string) {
 
 	names, err := crawl(w, w.baseURL, w.domain, sn, domain)
 	if err != nil {
-		w.Config().Log.Printf("%s: %v", w.String(), err)
+		w.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %v", w.String(), err))
 		return
 	}
 
diff --git a/services/sources/yahoo.go b/services/sources/yahoo.go
index e997e4f55..7f42057cd 100644
--- a/services/sources/yahoo.go
+++ b/services/sources/yahoo.go
@@ -4,6 +4,7 @@
 package sources
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"time"
@@ -26,14 +27,14 @@ type Yahoo struct {
 }
 
 // NewYahoo returns he object initialized, but not yet started.
-func NewYahoo(c *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Yahoo {
+func NewYahoo(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *Yahoo {
 	y := &Yahoo{
 		quantity:   10,
 		limit:      100,
 		SourceType: requests.SCRAPE,
 	}
 
-	y.BaseService = *services.NewBaseService(y, "Yahoo", c, bus, pool)
+	y.BaseService = *services.NewBaseService(y, "Yahoo", cfg, bus, pool)
 	return y
 }
 
@@ -81,7 +82,7 @@ func (y *Yahoo) executeQuery(domain string) {
 			u := y.urlByPageNum(domain, i)
 			page, err := utils.RequestWebPage(u, nil, nil, "", "")
 			if err != nil {
-				y.Config().Log.Printf("%s: %s: %v", y.String(), u, err)
+				y.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", y.String(), u, err))
 				return
 			}
 

From 42843dd56d0a689cf73bb7b3bca36285db9b17f7 Mon Sep 17 00:00:00 2001
From: Kian Jamali <kian.jamali@citrix.com>
Date: Mon, 29 Jul 2019 00:35:01 -0700
Subject: [PATCH 16/33] create hackerone_test.go

---
 services/sources/hackerone_test.go | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 services/sources/hackerone_test.go

diff --git a/services/sources/hackerone_test.go b/services/sources/hackerone_test.go
new file mode 100644
index 000000000..e69de29bb

From 7cb32b5f83f7d0020b525533796fa555fe932a19 Mon Sep 17 00:00:00 2001
From: Kian <kianjamali123@gmail.com>
Date: Mon, 29 Jul 2019 02:15:01 -0700
Subject: [PATCH 17/33] implemented the TestHackerone function

---
 services/sources/hackerone_test.go | 31 ++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/services/sources/hackerone_test.go b/services/sources/hackerone_test.go
index e69de29bb..a015c7223 100644
--- a/services/sources/hackerone_test.go
+++ b/services/sources/hackerone_test.go
@@ -0,0 +1,31 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+)
+
+func TestHackerone(t *testing.T) {
+	if *networkTest == false || *configPath == "" {
+		return
+	}
+
+	domainTest = "twitter.com"
+
+	cfg := setupConfig(domainTest)
+	
+	bus, out := setupEventBus(requests.NewNameTopic)
+	defer bus.Stop()
+
+	pool := resolvers.NewResolverPool(nil)
+	defer pool.Stop()
+
+	srv := NewHackerOne(cfg, bus, pool)
+
+	result := testService(srv, out)
+	if result < expectedTest {
+		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
+	}
+}
\ No newline at end of file

From 59856f348f5082073c30989697650a75a25dfc34 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Mon, 29 Jul 2019 20:59:04 -0400
Subject: [PATCH 18/33] moved CommonCrawl to the list of APIs

---
 README.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 07fe93e43..e4d4706b8 100644
--- a/README.md
+++ b/README.md
@@ -27,9 +27,9 @@ The OWASP Amass Project has developed a tool to help information security profes
 **Information Gathering Techniques Used:**
 
 * **DNS:** Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request)
-* **Scraping:** Ask, Baidu, Bing, CommonCrawl, DNSDumpster, DNSTable, Dogpile, Exalead, Google, HackerOne, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ViewDNS, Yahoo
+* **Scraping:** Ask, Baidu, Bing, DNSDumpster, DNSTable, Dogpile, Exalead, Google, HackerOne, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ViewDNS, Yahoo
 * **Certificates:** Active pulls (upon request), Censys, CertSpotter, Crtsh, Entrust, GoogleCT
-* **APIs:** AlienVault, BinaryEdge, BufferOver, CIRCL, DNSDB, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse (CertDB & FindSubdomains), Sublist3rAPI, TeamCymru, ThreatCrowd, Twitter, Umbrella, URLScan, VirusTotal
+* **APIs:** AlienVault, BinaryEdge, BufferOver, CIRCL, CommonCrawl, DNSDB, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse (CertDB & FindSubdomains), Sublist3rAPI, TeamCymru, ThreatCrowd, Twitter, Umbrella, URLScan, VirusTotal
 * **Web Archives:** ArchiveIt, ArchiveToday, Arquivo, LoCArchive, OpenUKArchive, UKGovArchive, Wayback
 
 ----

From b23b52402dca06fdff55e783b2779cf72882d138 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Mon, 29 Jul 2019 22:57:52 -0400
Subject: [PATCH 19/33] Replace UniqueAppend with set implementation

---
 cmd/amass/db.go                  |  21 +++--
 cmd/amass/enum.go                | 131 +++++++++++++------------------
 cmd/amass/intel.go               |  30 ++++---
 cmd/amass/track.go               |  17 ++--
 cmd/amass/viz.go                 |  16 ++--
 config/config.go                 |  72 ++++++++++-------
 enum/enum.go                     |   3 +-
 graph/graph.go                   |  20 ++---
 intel/intel.go                   |   6 +-
 requests/request.go              |   4 +-
 services/addrsrv.go              |   8 +-
 services/alteration.go           |   5 +-
 services/alteration_test.go      |   3 +-
 services/brute.go                |   7 +-
 services/sources/alienvault.go   |  47 +++++------
 services/sources/circl.go        |   7 +-
 services/sources/crtsh.go        |   8 +-
 services/sources/dnsdb.go        |  40 +++++-----
 services/sources/hackertarget.go |   3 +-
 services/sources/mnemonic.go     |  13 +--
 services/sources/networksdb.go   |  35 +++++----
 services/sources/radb.go         |  15 ++--
 services/sources/robtex.go       |  28 +++----
 services/sources/shadowserver.go |  15 ++--
 services/sources/sources.go      |   9 ++-
 services/sources/teamcymru.go    |   3 +-
 services/sources/umbrella.go     |  31 ++++----
 services/sources/urlscan.go      |  13 +--
 services/sources/viewdns.go      |   7 +-
 utils/misc.go                    |   5 --
 utils/misc_test.go               |  22 ------
 utils/parse.go                   |  30 +++++++
 utils/web.go                     |   9 ++-
 33 files changed, 353 insertions(+), 330 deletions(-)

diff --git a/cmd/amass/db.go b/cmd/amass/db.go
index 87a3b8f04..1405076b5 100644
--- a/cmd/amass/db.go
+++ b/cmd/amass/db.go
@@ -16,6 +16,7 @@ import (
 	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/graph"
 	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/fatih/color"
 )
@@ -25,7 +26,7 @@ const (
 )
 
 type dbArgs struct {
-	Domains utils.ParseStrings
+	Domains stringset.Set
 	Enum    int
 	Options struct {
 		DemoMode         bool
@@ -52,6 +53,8 @@ func runDBCommand(clArgs []string) {
 	dbBuf := new(bytes.Buffer)
 	dbCommand.SetOutput(dbBuf)
 
+	args.Domains = stringset.New()
+
 	dbCommand.BoolVar(&help1, "h", false, "Show the program usage message")
 	dbCommand.BoolVar(&help2, "help", false, "Show the program usage message")
 	dbCommand.Var(&args.Domains, "d", "Domain names separated by commas (can be used multiple times)")
@@ -88,17 +91,17 @@ func runDBCommand(clArgs []string) {
 			r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err)
 			return
 		}
-		args.Domains = utils.UniqueAppend(args.Domains, list...)
+		args.Domains.InsertMany(list...)
 	}
 
-	cfg := new(config.Config)
+	cfg := config.New()
 	// Check if a configuration file was provided, and if so, load the settings
 	if _, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {
 		if args.Filepaths.Directory == "" {
 			args.Filepaths.Directory = cfg.Dir
 		}
 		if len(args.Domains) == 0 {
-			args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...)
+			args.Domains.InsertMany(cfg.Domains()...)
 		}
 	} else if args.Filepaths.ConfigFile != "" {
 		r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
@@ -122,7 +125,7 @@ func runDBCommand(clArgs []string) {
 	}
 
 	if args.Options.ListEnumerations {
-		listEnumerations(args.Domains, db)
+		listEnumerations(&args, db)
 		return
 	}
 
@@ -172,7 +175,8 @@ func inputDataOperations(args *dbArgs, db graph.DataHandler) error {
 	return nil
 }
 
-func listEnumerations(domains []string, db graph.DataHandler) {
+func listEnumerations(args *dbArgs, db graph.DataHandler) {
+	domains := args.Domains.ToSlice()
 	enums := enumIDs(domains, db)
 	if len(enums) == 0 {
 		r.Fprintln(color.Error, "No enumerations found within the provided scope")
@@ -198,11 +202,12 @@ func listEnumerations(domains []string, db graph.DataHandler) {
 }
 
 func showEnumeration(args *dbArgs, db graph.DataHandler) {
+	domains := args.Domains.ToSlice()
 	var total int
 	tags := make(map[string]int)
 	asns := make(map[int]*utils.ASNSummaryData)
-	for _, out := range getEnumOutput(args.Enum, args.Domains, db) {
-		if len(args.Domains) > 0 && !domainNameInScope(out.Name, args.Domains) {
+	for _, out := range getEnumOutput(args.Enum, domains, db) {
+		if len(domains) > 0 && !domainNameInScope(out.Name, domains) {
 			continue
 		}
 
diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go
index 92ea30e63..7b382e3bb 100644
--- a/cmd/amass/enum.go
+++ b/cmd/amass/enum.go
@@ -23,6 +23,7 @@ import (
 
 	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/enum"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/fatih/color"
 	homedir "github.com/mitchellh/go-homedir"
@@ -40,19 +41,19 @@ type enumArgs struct {
 	Addresses         utils.ParseIPs
 	ASNs              utils.ParseInts
 	CIDRs             utils.ParseCIDRs
-	AltWordList       []string
-	AltWordListMask   utils.ParseStrings
-	BruteWordList     []string
-	BruteWordListMask utils.ParseStrings
-	Blacklist         utils.ParseStrings
-	Domains           utils.ParseStrings
-	Excluded          utils.ParseStrings
-	Included          utils.ParseStrings
+	AltWordList       stringset.Set
+	AltWordListMask   stringset.Set
+	BruteWordList     stringset.Set
+	BruteWordListMask stringset.Set
+	Blacklist         stringset.Set
+	Domains           stringset.Set
+	Excluded          stringset.Set
+	Included          stringset.Set
 	MaxDNSQueries     int
 	MinForRecursive   int
-	Names             []string
+	Names             stringset.Set
 	Ports             utils.ParseInts
-	Resolvers         utils.ParseStrings
+	Resolvers         stringset.Set
 	Options           struct {
 		Active       bool
 		BruteForcing bool
@@ -136,7 +137,18 @@ func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) {
 }
 
 func runEnumCommand(clArgs []string) {
-	var args enumArgs
+	args := enumArgs{
+		AltWordList:       stringset.New(),
+		AltWordListMask:   stringset.New(),
+		BruteWordList:     stringset.New(),
+		BruteWordListMask: stringset.New(),
+		Blacklist:         stringset.New(),
+		Domains:           stringset.New(),
+		Excluded:          stringset.New(),
+		Included:          stringset.New(),
+		Names:             stringset.New(),
+		Resolvers:         stringset.New(),
+	}
 	var help1, help2 bool
 	enumCommand := flag.NewFlagSet("enum", flag.ContinueOnError)
 
@@ -171,10 +183,10 @@ func runEnumCommand(clArgs []string) {
 	}
 
 	if len(args.AltWordListMask) > 0 {
-		args.AltWordList = utils.UniqueAppend(args.AltWordList, args.AltWordListMask...)
+		args.AltWordList.Union(args.AltWordListMask)
 	}
 	if len(args.BruteWordListMask) > 0 {
-		args.BruteWordList = utils.UniqueAppend(args.BruteWordList, args.BruteWordListMask...)
+		args.BruteWordList.Union(args.BruteWordListMask)
 	}
 	// Some input validation
 	if args.Options.Passive && (args.Options.IPs || args.Options.IPv4 || args.Options.IPv6) {
@@ -207,7 +219,7 @@ func runEnumCommand(clArgs []string) {
 	if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, e.Config); err == nil {
 		// Check if a config file was provided that has DNS resolvers specified
 		if r, err := config.GetResolversFromSettings(f); err == nil && len(args.Resolvers) == 0 {
-			args.Resolvers = r
+			args.Resolvers = stringset.New(r...)
 		}
 	} else if args.Filepaths.ConfigFile != "" {
 		r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
@@ -221,7 +233,7 @@ func runEnumCommand(clArgs []string) {
 	}
 
 	if len(args.Resolvers) > 0 {
-		if err := e.Pool.SetResolvers(args.Resolvers); err != nil {
+		if err := e.Pool.SetResolvers(args.Resolvers.ToSlice()); err != nil {
 			r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err)
 			os.Exit(1)
 		}
@@ -435,7 +447,7 @@ func processEnumInputFiles(args *enumArgs) error {
 				return fmt.Errorf("Failed to parse the brute force wordlist file: %v", err)
 			}
 
-			args.BruteWordList = utils.UniqueAppend(args.BruteWordList, list...)
+			args.BruteWordList.InsertMany(list...)
 		}
 	}
 	if !args.Options.NoAlts && len(args.Filepaths.AltWordlist) > 0 {
@@ -445,7 +457,7 @@ func processEnumInputFiles(args *enumArgs) error {
 				return fmt.Errorf("Failed to parse the alterations wordlist file: %v", err)
 			}
 
-			args.AltWordList = utils.UniqueAppend(args.AltWordList, list...)
+			args.AltWordList.InsertMany(list...)
 		}
 	}
 	if args.Filepaths.Blacklist != "" {
@@ -453,21 +465,21 @@ func processEnumInputFiles(args *enumArgs) error {
 		if err != nil {
 			return fmt.Errorf("Failed to parse the blacklist file: %v", err)
 		}
-		args.Blacklist = utils.UniqueAppend(args.Blacklist, list...)
+		args.Blacklist.InsertMany(list...)
 	}
 	if args.Filepaths.ExcludedSrcs != "" {
 		list, err := config.GetListFromFile(args.Filepaths.ExcludedSrcs)
 		if err != nil {
 			return fmt.Errorf("Failed to parse the exclude file: %v", err)
 		}
-		args.Excluded = utils.UniqueAppend(args.Excluded, list...)
+		args.Excluded.InsertMany(list...)
 	}
 	if args.Filepaths.IncludedSrcs != "" {
 		list, err := config.GetListFromFile(args.Filepaths.IncludedSrcs)
 		if err != nil {
 			return fmt.Errorf("Failed to parse the include file: %v", err)
 		}
-		args.Included = utils.UniqueAppend(args.Included, list...)
+		args.Included.InsertMany(list...)
 	}
 	if len(args.Filepaths.Names) > 0 {
 		for _, f := range args.Filepaths.Names {
@@ -476,7 +488,7 @@ func processEnumInputFiles(args *enumArgs) error {
 				return fmt.Errorf("Failed to parse the subdomain names file: %v", err)
 			}
 
-			args.Names = utils.UniqueAppend(args.Names, list...)
+			args.Names.InsertMany(list...)
 		}
 	}
 	if len(args.Filepaths.Domains) > 0 {
@@ -486,7 +498,7 @@ func processEnumInputFiles(args *enumArgs) error {
 				return fmt.Errorf("Failed to parse the domain names file: %v", err)
 			}
 
-			args.Domains = utils.UniqueAppend(args.Domains, list...)
+			args.Domains.InsertMany(list...)
 		}
 	}
 	if len(args.Filepaths.Resolvers) > 0 {
@@ -496,7 +508,7 @@ func processEnumInputFiles(args *enumArgs) error {
 				return fmt.Errorf("Failed to parse the resolver file: %v", err)
 			}
 
-			args.Resolvers = utils.UniqueAppend(args.Resolvers, list...)
+			args.Resolvers.InsertMany(list...)
 		}
 	}
 	return nil
@@ -529,7 +541,7 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
 		e.Config.AltWordlist = args.AltWordList
 	}
 	if len(args.Names) > 0 {
-		e.ProvidedNames = args.Names
+		e.ProvidedNames = args.Names.ToSlice()
 	}
 	if args.Options.BruteForcing {
 		e.Config.BruteForcing = true
@@ -562,71 +574,34 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
 	}
 
 	// Attempt to add the provided domains to the configuration
-	e.Config.AddDomains(args.Domains)
+	e.Config.AddDomains(args.Domains.ToSlice())
 	if len(e.Config.Domains()) == 0 {
 		return errors.New("No root domain names were provided")
 	}
 	return nil
 }
 
-func compileDisabledSources(srcs []string, include, exclude []string) []string {
-	var inc, disable []string
+func compileDisabledSources(srcs []string, include, exclude stringset.Set) stringset.Set {
+	master := stringset.New(srcs...)
 
-	master := srcs
-	// Check that the include names are valid
-	if len(include) > 0 {
-		for _, incname := range include {
-			var found bool
-
-			for _, name := range master {
-				if strings.EqualFold(name, incname) {
-					found = true
-					inc = append(inc, incname)
-					break
-				}
-			}
-
-			if !found {
-				r.Fprintf(color.Error, "%s is not an available data source\n", incname)
-			}
-		}
-	}
 	// Check that the exclude names are valid
-	if len(exclude) > 0 {
-		for _, exclname := range exclude {
-			var found bool
-
-			for _, name := range master {
-				if strings.EqualFold(name, exclname) {
-					found = true
-					disable = append(disable, exclname)
-					break
-				}
-			}
-
-			if !found {
-				r.Fprintf(color.Error, "%s is not an available data source\n", exclname)
-			}
-		}
+	excLen := len(exclude)
+	exclude.Intersect(master)
+	if excLen != len(exclude) {
+		r.Fprintf(color.Error, "Invalid excluded data source specification\n")
 	}
 
-	if len(inc) == 0 {
-		return disable
+	// Check that the include names are valid
+	incLen := len(include)
+	include.Intersect(master)
+	if incLen != len(include) {
+		r.Fprintf(color.Error, "Invalid included data source specification\n")
 	}
-	// Data sources missing from the include list are disabled
-	for _, name := range master {
-		var found bool
-
-		for _, incname := range inc {
-			if strings.EqualFold(name, incname) {
-				found = true
-				break
-			}
-		}
 
-		if !found {
-			disable = utils.UniqueAppend(disable, name)
-		}
+	if len(include) == 0 {
+		return exclude
 	}
-	return disable
+
+	master.Subtract(include)
+	return master
 }
diff --git a/cmd/amass/intel.go b/cmd/amass/intel.go
index 73f46edb9..317df9abd 100644
--- a/cmd/amass/intel.go
+++ b/cmd/amass/intel.go
@@ -19,6 +19,7 @@ import (
 
 	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/intel"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/fatih/color"
 	homedir "github.com/mitchellh/go-homedir"
@@ -33,12 +34,12 @@ type intelArgs struct {
 	ASNs             utils.ParseInts
 	CIDRs            utils.ParseCIDRs
 	OrganizationName string
-	Domains          utils.ParseStrings
-	Excluded         utils.ParseStrings
-	Included         utils.ParseStrings
+	Domains          stringset.Set
+	Excluded         stringset.Set
+	Included         stringset.Set
 	MaxDNSQueries    int
 	Ports            utils.ParseInts
-	Resolvers        utils.ParseStrings
+	Resolvers        stringset.Set
 	Options          struct {
 		Active       bool
 		DemoMode     bool
@@ -97,7 +98,12 @@ func defineIntelFilepathFlags(intelFlags *flag.FlagSet, args *intelArgs) {
 }
 
 func runIntelCommand(clArgs []string) {
-	var args intelArgs
+	args := intelArgs{
+		Domains:   stringset.New(),
+		Excluded:  stringset.New(),
+		Included:  stringset.New(),
+		Resolvers: stringset.New(),
+	}
 	var help1, help2 bool
 	intelCommand := flag.NewFlagSet("intel", flag.ContinueOnError)
 
@@ -172,7 +178,7 @@ func runIntelCommand(clArgs []string) {
 	if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, ic.Config); err == nil {
 		// Check if a config file was provided that has DNS resolvers specified
 		if r, err := config.GetResolversFromSettings(f); err == nil && len(args.Resolvers) == 0 {
-			args.Resolvers = r
+			args.Resolvers = stringset.New(r...)
 		}
 	} else if args.Filepaths.ConfigFile != "" {
 		r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
@@ -186,7 +192,7 @@ func runIntelCommand(clArgs []string) {
 	}
 
 	if len(args.Resolvers) > 0 {
-		if err := ic.Pool.SetResolvers(args.Resolvers); err != nil {
+		if err := ic.Pool.SetResolvers(args.Resolvers.ToSlice()); err != nil {
 			r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err)
 			os.Exit(1)
 		}
@@ -319,14 +325,14 @@ func processIntelInputFiles(args *intelArgs) error {
 		if err != nil {
 			return fmt.Errorf("Failed to parse the exclude file: %v", err)
 		}
-		args.Excluded = utils.UniqueAppend(args.Excluded, list...)
+		args.Excluded.InsertMany(list...)
 	}
 	if args.Filepaths.IncludedSrcs != "" {
 		list, err := config.GetListFromFile(args.Filepaths.IncludedSrcs)
 		if err != nil {
 			return fmt.Errorf("Failed to parse the include file: %v", err)
 		}
-		args.Included = utils.UniqueAppend(args.Included, list...)
+		args.Included.InsertMany(list...)
 	}
 	if len(args.Filepaths.Domains) > 0 {
 		for _, f := range args.Filepaths.Domains {
@@ -335,7 +341,7 @@ func processIntelInputFiles(args *intelArgs) error {
 				return fmt.Errorf("Failed to parse the domain names file: %v", err)
 			}
 
-			args.Domains = utils.UniqueAppend(args.Domains, list...)
+			args.Domains.InsertMany(list...)
 		}
 	}
 	if len(args.Filepaths.Resolvers) > 0 {
@@ -345,7 +351,7 @@ func processIntelInputFiles(args *intelArgs) error {
 				return fmt.Errorf("Failed to parse the resolver file: %v", err)
 			}
 
-			args.Resolvers = utils.UniqueAppend(args.Resolvers, list...)
+			args.Resolvers.InsertMany(list...)
 		}
 	}
 	return nil
@@ -380,6 +386,6 @@ func updateIntelConfiguration(ic *intel.Collection, args *intelArgs) error {
 		ic.Config.DisabledDataSources = disabled
 	}
 	// Attempt to add the provided domains to the configuration
-	ic.Config.AddDomains(args.Domains)
+	ic.Config.AddDomains(args.Domains.ToSlice())
 	return nil
 }
diff --git a/cmd/amass/track.go b/cmd/amass/track.go
index 317ce1ab4..928e52111 100644
--- a/cmd/amass/track.go
+++ b/cmd/amass/track.go
@@ -14,6 +14,7 @@ import (
 	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/graph"
 	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/fatih/color"
 )
@@ -24,7 +25,7 @@ const (
 )
 
 type trackArgs struct {
-	Domains utils.ParseStrings
+	Domains stringset.Set
 	Last    int
 	Since   string
 	Options struct {
@@ -42,6 +43,8 @@ func runTrackCommand(clArgs []string) {
 	var help1, help2 bool
 	trackCommand := flag.NewFlagSet("track", flag.ContinueOnError)
 
+	args.Domains = stringset.New()
+
 	trackBuf := new(bytes.Buffer)
 	trackCommand.SetOutput(trackBuf)
 
@@ -84,7 +87,7 @@ func runTrackCommand(clArgs []string) {
 			r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err)
 			os.Exit(1)
 		}
-		args.Domains = utils.UniqueAppend(args.Domains, list...)
+		args.Domains.InsertMany(list...)
 	}
 	if len(args.Domains) == 0 {
 		r.Fprintln(color.Error, "No root domain names were provided")
@@ -103,14 +106,14 @@ func runTrackCommand(clArgs []string) {
 
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	cfg := new(config.Config)
+	cfg := config.New()
 	// Check if a configuration file was provided, and if so, load the settings
 	if _, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {
 		if args.Filepaths.Directory == "" {
 			args.Filepaths.Directory = cfg.Dir
 		}
 		if len(args.Domains) == 0 {
-			args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...)
+			args.Domains.InsertMany(cfg.Domains()...)
 		}
 	} else if args.Filepaths.ConfigFile != "" {
 		r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
@@ -126,7 +129,7 @@ func runTrackCommand(clArgs []string) {
 	defer db.Close()
 
 	// Obtain the enumerations that include the provided domain(s)
-	enums := enumIDs(args.Domains, db)
+	enums := enumIDs(args.Domains.ToSlice(), db)
 
 	// There needs to be at least two enumerations to proceed
 	if len(enums) < 2 {
@@ -161,10 +164,10 @@ func runTrackCommand(clArgs []string) {
 	latest = latest[:end]
 
 	if args.Options.History {
-		completeHistoryOutput(args.Domains, enums, earliest, latest, db)
+		completeHistoryOutput(args.Domains.ToSlice(), enums, earliest, latest, db)
 		return
 	}
-	cumulativeOutput(args.Domains, enums, earliest, latest, db)
+	cumulativeOutput(args.Domains.ToSlice(), enums, earliest, latest, db)
 }
 
 func cumulativeOutput(domains []string, enums []string, ea, la []time.Time, db graph.DataHandler) {
diff --git a/cmd/amass/viz.go b/cmd/amass/viz.go
index ac2c28f63..cc751bc06 100644
--- a/cmd/amass/viz.go
+++ b/cmd/amass/viz.go
@@ -16,7 +16,7 @@ import (
 
 	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/graph"
-	"github.com/OWASP/Amass/utils"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils/viz"
 	"github.com/fatih/color"
 )
@@ -26,7 +26,7 @@ const (
 )
 
 type vizArgs struct {
-	Domains utils.ParseStrings
+	Domains stringset.Set
 	Enum    int
 	Options struct {
 		D3         bool
@@ -49,6 +49,8 @@ func runVizCommand(clArgs []string) {
 	var help1, help2 bool
 	vizCommand := flag.NewFlagSet("viz", flag.ContinueOnError)
 
+	args.Domains = stringset.New()
+
 	vizBuf := new(bytes.Buffer)
 	vizCommand.SetOutput(vizBuf)
 
@@ -94,7 +96,7 @@ func runVizCommand(clArgs []string) {
 			r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err)
 			return
 		}
-		args.Domains = utils.UniqueAppend(args.Domains, list...)
+		args.Domains.InsertMany(list...)
 	}
 
 	if args.Filepaths.Output == "" {
@@ -124,14 +126,14 @@ func runVizCommand(clArgs []string) {
 			defer db.Close()
 		}
 	} else {
-		cfg := new(config.Config)
+		cfg := config.New()
 		// Check if a configuration file was provided, and if so, load the settings
 		if _, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {
 			if args.Filepaths.Directory == "" {
 				args.Filepaths.Directory = cfg.Dir
 			}
 			if len(args.Domains) == 0 {
-				args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...)
+				args.Domains.InsertMany(cfg.Domains()...)
 			}
 		} else if args.Filepaths.ConfigFile != "" {
 			r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
@@ -146,10 +148,10 @@ func runVizCommand(clArgs []string) {
 		defer db.Close()
 
 		if args.Enum > 0 {
-			uuid = enumIndexToID(args.Enum, args.Domains, db)
+			uuid = enumIndexToID(args.Enum, args.Domains.ToSlice(), db)
 		} else {
 			// Get the UUID for the most recent enumeration
-			uuid = mostRecentEnumID(args.Domains, db)
+			uuid = mostRecentEnumID(args.Domains.ToSlice(), db)
 		}
 	}
 	if uuid == "" {
diff --git a/config/config.go b/config/config.go
index e0e56a594..4a87ca9f6 100644
--- a/config/config.go
+++ b/config/config.go
@@ -8,7 +8,6 @@ import (
 	"compress/gzip"
 	"errors"
 	"fmt"
-	homedir "github.com/mitchellh/go-homedir"
 	"io"
 	"log"
 	"net"
@@ -20,9 +19,11 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/go-ini/ini"
 	"github.com/google/uuid"
+	homedir "github.com/mitchellh/go-homedir"
 )
 
 const (
@@ -74,7 +75,7 @@ type Config struct {
 	Ports []int
 
 	// The list of words to use when generating names
-	Wordlist []string
+	Wordlist stringset.Set
 
 	// Will the enumeration including brute forcing techniques
 	BruteForcing bool
@@ -93,7 +94,7 @@ type Config struct {
 	AddNumbers     bool
 	MinForWordFlip int
 	EditDistance   int
-	AltWordlist    []string
+	AltWordlist    stringset.Set
 
 	// Only access the data sources for names and return results?
 	Passive bool
@@ -105,13 +106,13 @@ type Config struct {
 	IncludeUnresolvable bool `ini:"include_unresolvable"`
 
 	// A blacklist of subdomain names that will not be investigated
-	Blacklist []string
+	Blacklist stringset.Set
 
 	// A list of data sources that should not be utilized
-	DisabledDataSources []string
+	DisabledDataSources stringset.Set
 
 	// The root domain names that the enumeration will target
-	domains []string
+	domains stringset.Set
 
 	// The regular expressions for the root domains added to the enumeration
 	regexps map[string]*regexp.Regexp
@@ -128,6 +129,22 @@ type APIKey struct {
 	Secret   string `ini:"secret"`
 }
 
+func New() *Config {
+	c := new(Config)
+	c.Init()
+	return c
+}
+
+func (c *Config) Init() {
+	c.AltWordlist = stringset.New()
+	c.Wordlist = stringset.New()
+	c.domains = stringset.New()
+	c.Blacklist = stringset.New()
+	c.DisabledDataSources = stringset.New()
+	c.regexps = make(map[string]*regexp.Regexp)
+	c.apikeys = make(map[string]*APIKey)
+}
+
 // CheckSettings runs some sanity checks on the configuration options selected.
 func (c *Config) CheckSettings() error {
 	var err error
@@ -161,15 +178,19 @@ func (c *Config) CheckSettings() error {
 	}
 	c.SemMaxDNSQueries = utils.NewSimpleSemaphore(c.MaxDNSQueries)
 
-	c.Wordlist, err = utils.ExpandMaskWordlist(c.Wordlist)
+	wordlist := c.Wordlist.ToSlice()
+	wordlist, err = utils.ExpandMaskWordlist(wordlist)
 	if err != nil {
 		return err
 	}
+	c.Wordlist = stringset.New(wordlist...)
 
-	c.AltWordlist, err = utils.ExpandMaskWordlist(c.AltWordlist)
+	altWordlist := c.AltWordlist.ToSlice()
+	altWordlist, err = utils.ExpandMaskWordlist(altWordlist)
 	if err != nil {
 		return err
 	}
+	c.AltWordlist = stringset.New(altWordlist...)
 
 	return err
 }
@@ -213,15 +234,11 @@ func (c *Config) AddDomain(domain string) {
 			return
 		}
 	}
-	// Check that the regular expression map has been initialized
-	if c.regexps == nil {
-		c.regexps = make(map[string]*regexp.Regexp)
-	}
 	// Create the regular expression for this domain
 	c.regexps[d] = utils.SubdomainRegex(d)
 	if c.regexps[d] != nil {
 		// Add the domain string to the list
-		c.domains = utils.UniqueAppend(c.domains, d)
+		c.domains.Insert(d)
 	}
 }
 
@@ -230,7 +247,7 @@ func (c *Config) Domains() []string {
 	c.Lock()
 	defer c.Unlock()
 
-	return c.domains
+	return c.domains.ToSlice()
 }
 
 // IsDomainInScope returns true if the DNS name in the parameter ends with a domain in the config list.
@@ -290,7 +307,7 @@ func (c *Config) Blacklisted(name string) bool {
 	var resp bool
 
 	n := strings.TrimSpace(name)
-	for _, bl := range c.Blacklist {
+	for bl := range c.Blacklist {
 		if match := strings.HasSuffix(n, bl); match {
 			resp = true
 			break
@@ -309,9 +326,6 @@ func (c *Config) AddAPIKey(source string, ak *APIKey) {
 		return
 	}
 
-	if c.apikeys == nil {
-		c.apikeys = make(map[string]*APIKey)
-	}
 	c.apikeys[strings.ToLower(idx)] = ak
 }
 
@@ -389,7 +403,7 @@ func (c *Config) loadBruteForceSettings(cfg *ini.File) error {
 			if err != nil {
 				return fmt.Errorf("Unable to load the file in the bruteforce wordlist_file setting: %s: %v", wordlist, err)
 			}
-			c.Wordlist = utils.UniqueAppend(c.Wordlist, list...)
+			c.Wordlist.InsertMany(list...)
 		}
 	}
 	return nil
@@ -419,7 +433,7 @@ func (c *Config) loadAlterationSettings(cfg *ini.File) error {
 			if err != nil {
 				return fmt.Errorf("Unable to load the file in the alterations wordlist_file setting: %s: %v", wordlist, err)
 			}
-			c.AltWordlist = utils.UniqueAppend(c.AltWordlist, list...)
+			c.AltWordlist.InsertMany(list...)
 		}
 	}
 	return nil
@@ -456,13 +470,11 @@ func (c *Config) LoadSettings(path string) error {
 	}
 	// Load up all the blacklisted subdomain names
 	if blacklisted, err := cfg.GetSection("blacklisted"); err == nil {
-		c.Blacklist = utils.UniqueAppend(c.Blacklist,
-			blacklisted.Key("subdomain").ValueWithShadows()...)
+		c.Blacklist.InsertMany(blacklisted.Key("subdomain").ValueWithShadows()...)
 	}
 	// Load up all the disabled data source names
 	if disabled, err := cfg.GetSection("disabled_data_sources"); err == nil {
-		c.DisabledDataSources = utils.UniqueAppend(
-			c.DisabledDataSources, disabled.Key("data_source").ValueWithShadows()...)
+		c.DisabledDataSources.InsertMany(disabled.Key("data_source").ValueWithShadows()...)
 	}
 	// Load up all the Gremlin Server settings
 	if gremlin, err := cfg.GetSection("gremlin"); err == nil {
@@ -603,10 +615,12 @@ func GetListFromFile(path string) ([]string, error) {
 		defer gzReader.Close()
 		reader = gzReader
 	}
-	return getWordList(reader)
+
+	s, err := getWordList(reader)
+	return s.ToSlice(), err
 }
 
-func getWordlistByURL(url string) ([]string, error) {
+func getWordlistByURL(url string) (stringset.Set, error) {
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
 		return nil, fmt.Errorf("Failed to obtain the wordlist at %s: %v", url, err)
@@ -614,15 +628,15 @@ func getWordlistByURL(url string) ([]string, error) {
 	return getWordList(strings.NewReader(page))
 }
 
-func getWordList(reader io.Reader) ([]string, error) {
-	var words []string
+func getWordList(reader io.Reader) (stringset.Set, error) {
+	words := stringset.New()
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		// Get the next word in the list
 		w := strings.TrimSpace(scanner.Text())
 		if err := scanner.Err(); err == nil && w != "" {
-			words = append(words, w)
+			words.Insert(w)
 		}
 	}
 	return words, nil
diff --git a/enum/enum.go b/enum/enum.go
index 0515aa2d2..910e59b3c 100644
--- a/enum/enum.go
+++ b/enum/enum.go
@@ -81,6 +81,7 @@ func NewEnumeration() *Enumeration {
 		filter:      utils.NewStringFilter(),
 		outputQueue: new(utils.Queue),
 	}
+	e.Config.Init()
 	if e.Pool == nil {
 		return nil
 	}
@@ -463,7 +464,7 @@ func ExcludeDisabledDataSources(srvs []services.Service, cfg *config.Config) []s
 	for _, s := range srvs {
 		include := true
 
-		for _, disabled := range cfg.DisabledDataSources {
+		for disabled := range cfg.DisabledDataSources {
 			if strings.EqualFold(disabled, s.String()) {
 				include = false
 				break
diff --git a/graph/graph.go b/graph/graph.go
index f6436349e..07d433fa7 100644
--- a/graph/graph.go
+++ b/graph/graph.go
@@ -16,7 +16,7 @@ import (
 
 	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
-	"github.com/OWASP/Amass/utils"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils/viz"
 	"github.com/cayleygraph/cayley"
 	"github.com/cayleygraph/cayley/graph"
@@ -445,7 +445,7 @@ func (g *Graph) EnumerationList() []string {
 	it, _ := p.BuildIterator().Optimize()
 	defer it.Close()
 
-	var ids []string
+	ids := stringset.New()
 	ctx := context.TODO()
 	for it.Next(ctx) {
 		token := it.Result()
@@ -453,10 +453,10 @@ func (g *Graph) EnumerationList() []string {
 		label := quad.NativeOf(value).(string)
 
 		if label != "" {
-			ids = utils.UniqueAppend(ids, label)
+			ids.Insert(label)
 		}
 	}
-	return ids
+	return ids.ToSlice()
 }
 
 // EnumerationDomains returns the domains that were involved in the provided enumeration.
@@ -469,7 +469,7 @@ func (g *Graph) EnumerationDomains(uuid string) []string {
 	it, _ := p.BuildIterator().Optimize()
 	defer it.Close()
 
-	var domains []string
+	domains := stringset.New()
 	ctx := context.TODO()
 	for it.Next(ctx) {
 		token := it.Result()
@@ -477,10 +477,10 @@ func (g *Graph) EnumerationDomains(uuid string) []string {
 		domain := quad.NativeOf(value).(string)
 
 		if domain != "" {
-			domains = utils.UniqueAppend(domains, domain)
+			domains.Insert(domain)
 		}
 	}
-	return domains
+	return domains.ToSlice()
 }
 
 // EnumerationDateRange returns the date range associated with the provided enumeration UUID.
@@ -588,7 +588,7 @@ func (g *Graph) getSubdomainNames(domain, uuid string, marked bool) []string {
 }
 
 func (g *Graph) getCNAMEs(sub, uuid string) []string {
-	names := []string{sub}
+	names := stringset.New(sub)
 
 	cname := quad.String(sub)
 	for i := 0; i < 10; i++ {
@@ -598,9 +598,9 @@ func (g *Graph) getCNAMEs(sub, uuid string) []string {
 		}
 		// Traverse to the next CNAME
 		cname = quad.String(target)
-		names = utils.UniqueAppend(names, target)
+		names.Insert(target)
 	}
-	return names
+	return names.ToSlice()
 }
 
 func (g *Graph) buildOutput(sub, uuid string) *requests.Output {
diff --git a/intel/intel.go b/intel/intel.go
index 8e007d8c7..397b4241c 100644
--- a/intel/intel.go
+++ b/intel/intel.go
@@ -234,7 +234,7 @@ func (c *Collection) sendNetblockCIDRs() {
 
 	filter := utils.NewStringFilter()
 	for _, record := range c.netCache {
-		for _, netblock := range record.Netblocks {
+		for netblock := range record.Netblocks {
 			_, ipnet, err := net.ParseCIDR(netblock)
 			if err == nil && !filter.Duplicate(ipnet.String()) {
 				c.cidrChan <- ipnet
@@ -269,7 +269,7 @@ func (c *Collection) updateNetCache(req *requests.ASNRequest) {
 	if entry.Description == "" && req.Description != "" {
 		entry.Description = req.Description
 	}
-	entry.Netblocks = utils.UniqueAppend(entry.Netblocks, req.Netblocks...)
+	entry.Netblocks.Union(req.Netblocks)
 	c.netCache[req.ASN] = entry
 }
 
@@ -380,7 +380,7 @@ func ExcludeDisabledDataSources(srvs []services.Service, cfg *config.Config) []s
 	for _, s := range srvs {
 		include := true
 
-		for _, disabled := range cfg.DisabledDataSources {
+		for disabled := range cfg.DisabledDataSources {
 			if strings.EqualFold(disabled, s.String()) {
 				include = false
 				break
diff --git a/requests/request.go b/requests/request.go
index 4bcf55ae8..1611a8c17 100644
--- a/requests/request.go
+++ b/requests/request.go
@@ -6,6 +6,8 @@ package requests
 import (
 	"net"
 	"time"
+
+	"github.com/OWASP/Amass/stringset"
 )
 
 // Request tag types.
@@ -71,7 +73,7 @@ type ASNRequest struct {
 	Registry       string
 	AllocationDate time.Time
 	Description    string
-	Netblocks      []string
+	Netblocks      stringset.Set
 	Tag            string
 	Source         string
 }
diff --git a/services/addrsrv.go b/services/addrsrv.go
index e170d51f8..a76c3bb1c 100644
--- a/services/addrsrv.go
+++ b/services/addrsrv.go
@@ -162,7 +162,7 @@ func (as *AddressService) updateConfigWithNetblocks(req *requests.ASNRequest) {
 		filter.Duplicate(cidr.String())
 	}
 
-	for _, block := range req.Netblocks {
+	for block := range req.Netblocks {
 		if filter.Duplicate(block) {
 			continue
 		}
@@ -198,7 +198,7 @@ loop:
 			break loop
 		case a := <-asnchan:
 			updateCache(a)
-			for _, block := range a.Netblocks {
+			for block := range a.Netblocks {
 				if _, cidr, err := net.ParseCIDR(block); err == nil && cidr.Contains(ip) {
 					break loop
 				}
@@ -242,7 +242,7 @@ func updateCache(req *requests.ASNRequest) {
 	if c.Description == "" && req.Description != "" {
 		c.Description = req.Description
 	}
-	c.Netblocks = utils.UniqueAppend(c.Netblocks, req.Netblocks...)
+	c.Netblocks.Union(req.Netblocks)
 	netCache[req.ASN] = c
 }
 
@@ -260,7 +260,7 @@ func ipSearch(addr string) *requests.ASNRequest {
 	var desc string
 	ip := net.ParseIP(addr)
 	for asn, record := range netCache {
-		for _, netblock := range record.Netblocks {
+		for netblock := range record.Netblocks {
 			_, ipnet, err := net.ParseCIDR(netblock)
 			if err != nil {
 				continue
diff --git a/services/alteration.go b/services/alteration.go
index b1e9a4780..b95cc8a3a 100644
--- a/services/alteration.go
+++ b/services/alteration.go
@@ -60,10 +60,11 @@ type AlterationService struct {
 
 // NewAlterationService returns he object initialized, but not yet started.
 func NewAlterationService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AlterationService {
+	seed := cfg.AltWordlist.ToSlice()
 	as := &AlterationService{
 		filter:   utils.NewStringFilter(),
-		prefixes: newAlterationCache(cfg.AltWordlist),
-		suffixes: newAlterationCache(cfg.AltWordlist),
+		prefixes: newAlterationCache(seed),
+		suffixes: newAlterationCache(seed),
 	}
 
 	as.BaseService = *NewBaseService(as, "Alterations", cfg, bus, pool)
diff --git a/services/alteration_test.go b/services/alteration_test.go
index 24d24fcd3..b6b68e33c 100644
--- a/services/alteration_test.go
+++ b/services/alteration_test.go
@@ -28,6 +28,7 @@ var (
 
 func setupConfig(domain string) *config.Config {
 	cfg := &config.Config{}
+	cfg.Init()
 	cfg.Alterations = true
 	cfg.FlipWords = true
 	cfg.AddWords = true
@@ -35,7 +36,7 @@ func setupConfig(domain string) *config.Config {
 	cfg.AddNumbers = true
 	cfg.MinForWordFlip = 0
 	cfg.EditDistance = 1
-	cfg.AltWordlist = []string{"prod", "dev"}
+	cfg.AltWordlist.InsertMany("prod", "dev")
 	cfg.AddDomain(domain)
 	buf := new(strings.Builder)
 	cfg.Log = log.New(buf, "", log.Lmicroseconds)
diff --git a/services/brute.go b/services/brute.go
index 83a44d3b2..7532a5876 100644
--- a/services/brute.go
+++ b/services/brute.go
@@ -145,9 +145,10 @@ func (bfs *BruteForceService) performBruteForcing(subdomain, domain string) {
 		bfs.Pool().GetWildcardType(req) == resolvers.WildcardTypeDynamic {
 		return
 	}
+	wordlist := bfs.Config().Wordlist.ToSlice()
 
 	bfs.totalLock.Lock()
-	bfs.totalNames += len(bfs.Config().Wordlist)
+	bfs.totalNames += len(wordlist)
 	bfs.totalLock.Unlock()
 
 	var idx int
@@ -160,11 +161,11 @@ func (bfs *BruteForceService) performBruteForcing(subdomain, domain string) {
 		case <-t.C:
 			bfs.SetActive()
 		default:
-			if idx >= len(bfs.Config().Wordlist) {
+			if idx >= len(wordlist) {
 				return
 			}
 			bfs.Config().SemMaxDNSQueries.Acquire(1)
-			word := strings.ToLower(bfs.Config().Wordlist[idx])
+			word := strings.ToLower(wordlist[idx])
 			go bfs.bruteForceResolution(word, subdomain, domain)
 			idx++
 		}
diff --git a/services/sources/alienvault.go b/services/sources/alienvault.go
index e39a67959..5bc451dad 100644
--- a/services/sources/alienvault.go
+++ b/services/sources/alienvault.go
@@ -16,6 +16,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -113,18 +114,18 @@ func (a *AlienVault) executeDNSQuery(domain string) {
 		return
 	}
 
-	var ips []string
-	var names []string
+	ips := stringset.New()
+	names := stringset.New()
 	for _, sub := range m.Subdomains {
 		n := strings.ToLower(sub.Hostname)
 
 		if re.MatchString(n) {
-			names = append(names, n)
-			ips = append(ips, sub.IP)
+			names.Insert(n)
+			ips.Insert(sub.IP)
 		}
 	}
 
-	for _, name := range names {
+	for name := range names {
 		a.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
@@ -133,7 +134,7 @@ func (a *AlienVault) executeDNSQuery(domain string) {
 		})
 	}
 
-	for _, ip := range ips {
+	for ip := range ips {
 		a.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{
 			Address: ip,
 			Tag:     a.SourceType,
@@ -180,15 +181,15 @@ func (a *AlienVault) executeURLQuery(domain string) {
 		return
 	}
 
-	var ips []string
-	var names []string
+	ips := stringset.New()
+	names := stringset.New()
 	for _, u := range urls.URLs {
 		n := strings.ToLower(u.Hostname)
 
 		if re.MatchString(n) {
-			names = utils.UniqueAppend(names, n)
+			names.Insert(n)
 			if u.Result.Worker.IP != "" {
-				ips = utils.UniqueAppend(ips, u.Result.Worker.IP)
+				ips.Insert(u.Result.Worker.IP)
 			}
 		}
 	}
@@ -218,16 +219,16 @@ func (a *AlienVault) executeURLQuery(domain string) {
 				n := strings.ToLower(u.Hostname)
 
 				if re.MatchString(n) {
-					names = utils.UniqueAppend(names, n)
+					names.Insert(n)
 					if u.Result.Worker.IP != "" {
-						ips = utils.UniqueAppend(ips, u.Result.Worker.IP)
+						ips.Insert(u.Result.Worker.IP)
 					}
 				}
 			}
 		}
 	}
 
-	for _, name := range names {
+	for name := range names {
 		a.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
@@ -236,7 +237,7 @@ func (a *AlienVault) executeURLQuery(domain string) {
 		})
 	}
 
-	for _, ip := range ips {
+	for ip := range ips {
 		a.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{
 			Address: ip,
 			Tag:     a.SourceType,
@@ -246,14 +247,14 @@ func (a *AlienVault) executeURLQuery(domain string) {
 }
 
 func (a *AlienVault) queryWhoisForEmails(domain string) []string {
-	var emails []string
+	emails := stringset.New()
 	u := a.getWhoisURL(domain)
 
 	a.SetActive()
 	page, err := utils.RequestWebPage(u, nil, a.getHeaders(), "", "")
 	if err != nil {
 		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
-		return emails
+		return emails.ToSlice()
 	}
 
 	var m struct {
@@ -266,12 +267,12 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
 		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
-		return emails
+		return emails.ToSlice()
 	} else if m.Count == 0 {
 		a.Bus().Publish(requests.LogTopic,
 			fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u),
 		)
-		return emails
+		return emails.ToSlice()
 	}
 
 	for _, row := range m.Data {
@@ -286,18 +287,18 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 			// Unfortunately AlienVault doesn't categorize the email addresses so we
 			// have to filter by something we know to avoid adding registrar emails
 			if a.Config().IsDomainInScope(d) {
-				emails = utils.UniqueAppend(emails, email)
+				emails.Insert(email)
 			}
 		}
 	}
-	return emails
+	return emails.ToSlice()
 }
 
 func (a *AlienVault) executeWhoisQuery(domain string) {
 	emails := a.queryWhoisForEmails(domain)
 	time.Sleep(a.RateLimit)
 
-	var newDomains []string
+	newDomains := stringset.New()
 	headers := a.getHeaders()
 	for _, email := range emails {
 		a.SetActive()
@@ -318,7 +319,7 @@ func (a *AlienVault) executeWhoisQuery(domain string) {
 		}
 		for _, d := range domains {
 			if !a.Config().IsDomainInScope(d.Domain) {
-				newDomains = utils.UniqueAppend(newDomains, d.Domain)
+				newDomains.Insert(d.Domain)
 			}
 		}
 		time.Sleep(a.RateLimit)
@@ -333,7 +334,7 @@ func (a *AlienVault) executeWhoisQuery(domain string) {
 
 	a.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{
 		Domain:     domain,
-		NewDomains: newDomains,
+		NewDomains: newDomains.ToSlice(),
 		Tag:        a.SourceType,
 		Source:     a.String(),
 	})
diff --git a/services/sources/circl.go b/services/sources/circl.go
index f2d5cdb94..ee6d7e878 100644
--- a/services/sources/circl.go
+++ b/services/sources/circl.go
@@ -15,6 +15,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -96,7 +97,7 @@ func (c *CIRCL) restURL(domain string) string {
 }
 
 func (c *CIRCL) passiveDNSJSON(page, domain string) {
-	var unique []string
+	unique := stringset.New()
 
 	re := c.Config().DomainRegex(domain)
 	if re == nil {
@@ -120,11 +121,11 @@ func (c *CIRCL) passiveDNSJSON(page, domain string) {
 			continue
 		}
 		if re.MatchString(j.Name) {
-			unique = utils.UniqueAppend(unique, j.Name)
+			unique.Insert(j.Name)
 		}
 	}
 
-	for _, name := range unique {
+	for name := range unique {
 		c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
diff --git a/services/sources/crtsh.go b/services/sources/crtsh.go
index 2250df3a3..1eeda84fe 100644
--- a/services/sources/crtsh.go
+++ b/services/sources/crtsh.go
@@ -6,13 +6,13 @@ package sources
 import (
 	"encoding/json"
 	"fmt"
-	"strings"
 
 	"github.com/OWASP/Amass/config"
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq" // Need the postgres driver
@@ -95,12 +95,12 @@ func (c *Crtsh) executeQuery(domain string) {
 
 	c.SetActive()
 	// Extract the subdomain names from the results
-	var names []string
+	names := stringset.New()
 	for _, result := range results {
-		names = utils.UniqueAppend(names, strings.ToLower(utils.RemoveAsteriskLabel(result.Domain)))
+		names.Insert(utils.RemoveAsteriskLabel(result.Domain))
 	}
 
-	for _, name := range names {
+	for name := range names {
 		c.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
diff --git a/services/sources/dnsdb.go b/services/sources/dnsdb.go
index 3423c52f1..cc6ed2322 100644
--- a/services/sources/dnsdb.go
+++ b/services/sources/dnsdb.go
@@ -16,6 +16,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -101,13 +102,13 @@ func (d *DNSDB) restURL(domain string) string {
 }
 
 func (d *DNSDB) passiveDNSJSON(page, domain string) {
-	var unique []string
 
 	re := d.Config().DomainRegex(domain)
 	if re == nil {
 		return
 	}
 
+	unique := stringset.New()
 	scanner := bufio.NewScanner(strings.NewReader(page))
 	for scanner.Scan() {
 		// Get the next line of JSON
@@ -124,11 +125,11 @@ func (d *DNSDB) passiveDNSJSON(page, domain string) {
 			continue
 		}
 		if re.MatchString(j.Name) {
-			unique = utils.UniqueAppend(unique, j.Name)
+			unique.Insert(j.Name)
 		}
 	}
 
-	for _, name := range unique {
+	for name := range unique {
 		d.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
@@ -146,14 +147,12 @@ func (d *DNSDB) scrape(domain string) {
 		return
 	}
 
-	var names []string
-	if f := d.followIndicies(page, domain); len(f) > 0 {
-		names = utils.UniqueAppend(names, f...)
-	} else if n := d.pullPageNames(page, domain); len(n) > 0 {
-		names = utils.UniqueAppend(names, n...)
-	}
+	names := stringset.New()
+	names.Union(d.followIndicies(page, domain))
+	names.Union(d.pullPageNames(page, domain))
+
 	// Share what has been discovered so far
-	for _, name := range names {
+	for name := range names {
 		d.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
@@ -165,7 +164,7 @@ func (d *DNSDB) scrape(domain string) {
 	t := time.NewTicker(d.RateLimit)
 	defer t.Stop()
 loop:
-	for _, name := range names {
+	for name := range names {
 		select {
 		case <-d.Quit():
 			break loop
@@ -181,7 +180,7 @@ loop:
 				continue
 			}
 
-			for _, result := range d.pullPageNames(another, domain) {
+			for result := range d.pullPageNames(another, domain) {
 				d.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 					Name:   result,
 					Domain: domain,
@@ -210,8 +209,9 @@ func (d *DNSDB) getURL(domain, sub string) string {
 
 var dnsdbIndexRE = regexp.MustCompile(`<a href="[a-zA-Z0-9]">([a-zA-Z0-9])</a>`)
 
-func (d *DNSDB) followIndicies(page, domain string) []string {
-	var indicies, unique []string
+func (d *DNSDB) followIndicies(page, domain string) stringset.Set {
+	var indicies []string
+	unique := stringset.New()
 	idx := dnsdbIndexRE.FindAllStringSubmatch(page, -1)
 	if idx == nil {
 		return unique
@@ -231,22 +231,18 @@ func (d *DNSDB) followIndicies(page, domain string) []string {
 			continue
 		}
 
-		if names := d.pullPageNames(ipage, domain); len(names) > 0 {
-			unique = utils.UniqueAppend(unique, names...)
-		}
+		unique.Union(d.pullPageNames(ipage, domain))
 		time.Sleep(d.RateLimit)
 	}
 	return unique
 }
 
-func (d *DNSDB) pullPageNames(page, domain string) []string {
-	var names []string
+func (d *DNSDB) pullPageNames(page, domain string) stringset.Set {
+	names := stringset.New()
 
 	if re := d.Config().DomainRegex(domain); re != nil {
 		for _, name := range re.FindAllString(page, -1) {
-			if u := utils.NewUniqueElements(names, cleanName(name)); len(u) > 0 {
-				names = append(names, u...)
-			}
+			names.Insert(cleanName(name))
 		}
 	}
 	return names
diff --git a/services/sources/hackertarget.go b/services/sources/hackertarget.go
index c3a3aa744..7a9cfd042 100644
--- a/services/sources/hackertarget.go
+++ b/services/sources/hackertarget.go
@@ -14,6 +14,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -119,7 +120,7 @@ func (h *HackerTarget) executeASNQuery(addr string) {
 		Prefix:         strings.Trim(fields[2], "\""),
 		AllocationDate: time.Now(),
 		Description:    strings.Trim(fields[3], "\""),
-		Netblocks:      []string{strings.Trim(fields[2], "\"")},
+		Netblocks:      stringset.New(strings.Trim(fields[2], "\"")),
 		Tag:            h.SourceType,
 		Source:         h.String(),
 	})
diff --git a/services/sources/mnemonic.go b/services/sources/mnemonic.go
index 8be2e9e37..d75aecc9c 100644
--- a/services/sources/mnemonic.go
+++ b/services/sources/mnemonic.go
@@ -14,6 +14,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -65,8 +66,8 @@ func (m *Mnemonic) executeDNSQuery(domain string) {
 		return
 	}
 
-	var ips []string
-	var names []string
+	ips := stringset.New()
+	names := stringset.New()
 	scanner := bufio.NewScanner(strings.NewReader(page))
 	for scanner.Scan() {
 		// Get the next line of JSON
@@ -85,12 +86,12 @@ func (m *Mnemonic) executeDNSQuery(domain string) {
 		}
 
 		if (j.Type == "a" || j.Type == "aaaa") && m.Config().IsDomainInScope(j.Query) {
-			ips = utils.UniqueAppend(ips, j.Answer)
-			names = utils.UniqueAppend(names, j.Query)
+			ips.Insert(j.Answer)
+			names.Insert(j.Query)
 		}
 	}
 
-	for _, name := range names {
+	for name := range names {
 		m.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 			Name:   name,
 			Domain: domain,
@@ -99,7 +100,7 @@ func (m *Mnemonic) executeDNSQuery(domain string) {
 		})
 	}
 
-	for _, ip := range ips {
+	for ip := range ips {
 		// Inform the Address Service of this finding
 		m.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{
 			Address: ip,
diff --git a/services/sources/networksdb.go b/services/sources/networksdb.go
index 30a191e4d..5c9d3bbae 100644
--- a/services/sources/networksdb.go
+++ b/services/sources/networksdb.go
@@ -18,6 +18,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -100,7 +101,7 @@ loop:
 				if req.Address != "" {
 					n.executeASNAddrQuery(req.Address)
 				} else {
-					n.executeASNQuery(req.ASN, "", []string{})
+					n.executeASNQuery(req.ASN, "", stringset.New())
 				}
 			}
 			last = time.Now()
@@ -135,10 +136,10 @@ func (n *NetworksDB) executeASNAddrQuery(addr string) {
 		return
 	}
 
-	var netblocks []string
+	netblocks := stringset.New()
 	for _, match := range networksdbCIDRRE.FindAllStringSubmatch(page, -1) {
 		if len(match) >= 2 {
-			netblocks = utils.UniqueAppend(netblocks, strings.TrimSpace(match[1]))
+			netblocks.Insert(strings.TrimSpace(match[1]))
 		}
 	}
 
@@ -167,7 +168,7 @@ func (n *NetworksDB) getIPURL(addr string) string {
 	return networksdbBaseURL + "/ip/" + addr
 }
 
-func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) {
+func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks stringset.Set) {
 	n.SetActive()
 	u := n.getASNURL(asn)
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
@@ -196,7 +197,7 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) {
 
 	for _, match := range networksdbCIDRRE.FindAllStringSubmatch(page, -1) {
 		if len(match) >= 2 {
-			netblocks = utils.UniqueAppend(netblocks, strings.TrimSpace(match[1]))
+			netblocks.Insert(strings.TrimSpace(match[1]))
 		}
 	}
 
@@ -204,7 +205,7 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) {
 	if addr != "" {
 		ip := net.ParseIP(addr)
 
-		for _, cidr := range netblocks {
+		for cidr := range netblocks {
 			if _, ipnet, err := net.ParseCIDR(cidr); err == nil && ipnet.Contains(ip) {
 				prefix = cidr
 				break
@@ -212,7 +213,7 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks []string) {
 		}
 	}
 	if prefix == "" && len(netblocks) > 0 {
-		prefix = netblocks[0]
+		prefix = netblocks.ToSlice()[0] // TODO order may matter here :shrug:
 	}
 
 	n.Bus().Publish(requests.NewASNTopic, &requests.ASNRequest{
@@ -250,7 +251,7 @@ func (n *NetworksDB) executeAPIASNAddrQuery(addr string) {
 	}
 
 	var asn int
-	var cidrs []string
+	cidrs := stringset.New()
 	ip := net.ParseIP(addr)
 loop:
 	for _, a := range asns {
@@ -262,7 +263,7 @@ loop:
 			)
 		}
 
-		for _, cidr := range cidrs {
+		for cidr := range cidrs {
 			if _, ipnet, err := net.ParseCIDR(cidr); err == nil {
 				if ipnet.Contains(ip) {
 					asn = a
@@ -281,9 +282,9 @@ loop:
 	n.executeAPIASNQuery(asn, addr, cidrs)
 }
 
-func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string) {
-	if netblocks == nil {
-		netblocks = n.apiNetblocksQuery(asn)
+func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks stringset.Set) {
+	if len(netblocks) == 0 {
+		netblocks.Union(n.apiNetblocksQuery(asn))
 		if len(netblocks) == 0 {
 			n.Bus().Publish(requests.LogTopic,
 				fmt.Sprintf("%s: %d: Failed to obtain netblocks associated with the ASN", n.String(), asn),
@@ -295,7 +296,7 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string
 	var prefix string
 	if addr != "" {
 		ip := net.ParseIP(addr)
-		for _, cidr := range netblocks {
+		for cidr := range netblocks {
 			if _, ipnet, err := net.ParseCIDR(cidr); err == nil && ipnet.Contains(ip) {
 				prefix = cidr
 				break
@@ -303,7 +304,7 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks []string
 		}
 	}
 	if prefix == "" {
-		prefix = netblocks[0]
+		prefix = netblocks.ToSlice()[0]
 	}
 
 	time.Sleep(n.RateLimit)
@@ -452,8 +453,8 @@ func (n *NetworksDB) getAPIASNInfoURL() string {
 	return networksdbBaseURL + networksdbAPIPATH + "/as/info"
 }
 
-func (n *NetworksDB) apiNetblocksQuery(asn int) []string {
-	var netblocks []string
+func (n *NetworksDB) apiNetblocksQuery(asn int) stringset.Set {
+	netblocks := stringset.New()
 
 	n.SetActive()
 	u := n.getAPINetblocksURL()
@@ -486,7 +487,7 @@ func (n *NetworksDB) apiNetblocksQuery(asn int) []string {
 	}
 
 	for _, block := range m.Results {
-		netblocks = utils.UniqueAppend(netblocks, block.CIDR)
+		netblocks.Insert(block.CIDR)
 	}
 	return netblocks
 }
diff --git a/services/sources/radb.go b/services/sources/radb.go
index 3b91c735e..f6905d492 100644
--- a/services/sources/radb.go
+++ b/services/sources/radb.go
@@ -18,6 +18,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -219,11 +220,11 @@ func (r *RADb) executeASNQuery(asn int, prefix string) {
 	r.SetActive()
 	time.Sleep(r.RateLimit)
 
-	var blocks []string
+	blocks := stringset.New()
 	if prefix != "" {
-		blocks = []string{prefix}
+		blocks.Insert(prefix)
 	}
-	blocks = utils.UniqueAppend(blocks, r.netblocks(asn)...)
+	blocks.Union(r.netblocks(asn))
 
 	if len(blocks) == 0 {
 		r.Bus().Publish(requests.LogTopic,
@@ -234,7 +235,7 @@ func (r *RADb) executeASNQuery(asn int, prefix string) {
 
 	r.Bus().Publish(requests.NewASNTopic, &requests.ASNRequest{
 		ASN:            asn,
-		Prefix:         blocks[0],
+		Prefix:         prefix,
 		AllocationDate: at,
 		Description:    m.Description,
 		Netblocks:      blocks,
@@ -249,8 +250,8 @@ func (r *RADb) getASNURL(registry, asn string) string {
 	return fmt.Sprintf(format, asn)
 }
 
-func (r *RADb) netblocks(asn int) []string {
-	var netblocks []string
+func (r *RADb) netblocks(asn int) stringset.Set {
+	netblocks := stringset.New()
 
 	r.SetActive()
 	url := r.getNetblocksURL(strconv.Itoa(asn))
@@ -295,7 +296,7 @@ func (r *RADb) netblocks(asn int) []string {
 			if prefix != "" {
 				l := strconv.Itoa(cidr.Length)
 
-				netblocks = utils.UniqueAppend(netblocks, prefix+"/"+l)
+				netblocks.Insert(prefix + "/" + l)
 			}
 		}
 	}
diff --git a/services/sources/robtex.go b/services/sources/robtex.go
index 8bee33a70..a074d3b45 100644
--- a/services/sources/robtex.go
+++ b/services/sources/robtex.go
@@ -17,6 +17,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -91,8 +92,6 @@ loop:
 }
 
 func (r *Robtex) executeDNSQuery(domain string) {
-	var ips []string
-
 	re := r.Config().DomainRegex(domain)
 	if re == nil {
 		return
@@ -106,9 +105,10 @@ func (r *Robtex) executeDNSQuery(domain string) {
 		return
 	}
 
+	ips := stringset.New()
 	for _, line := range r.parseDNSJSON(page) {
 		if line.Type == "A" {
-			ips = utils.UniqueAppend(ips, line.Data)
+			ips.Insert(line.Data)
 			// Inform the Address Service of this finding
 			r.Bus().Publish(requests.NewAddrTopic, &requests.AddrRequest{
 				Address: line.Data,
@@ -119,11 +119,11 @@ func (r *Robtex) executeDNSQuery(domain string) {
 		}
 	}
 
-	var names []string
+	names := stringset.New()
 	t := time.NewTicker(500 * time.Millisecond)
 	defer t.Stop()
 loop:
-	for _, ip := range ips {
+	for ip := range ips {
 		r.SetActive()
 
 		select {
@@ -138,12 +138,12 @@ loop:
 			}
 
 			for _, line := range r.parseDNSJSON(pdns) {
-				names = utils.UniqueAppend(names, line.Name)
+				names.Insert(line.Name)
 			}
 		}
 	}
 
-	for _, name := range names {
+	for name := range names {
 		if r.Config().IsDomainInScope(name) {
 			r.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 				Name:   name,
@@ -183,7 +183,7 @@ func (r *Robtex) executeASNQuery(asn int) {
 		return
 	}
 
-	_, ipnet, err := net.ParseCIDR(blocks[0])
+	_, ipnet, err := net.ParseCIDR(blocks.ToSlice()[0])
 	if err != nil {
 		return
 	}
@@ -195,7 +195,7 @@ func (r *Robtex) executeASNQuery(asn int) {
 		return
 	}
 
-	req.Netblocks = utils.UniqueAppend(req.Netblocks, blocks...)
+	req.Netblocks.Union(blocks)
 	r.Bus().Publish(requests.NewASNTopic, req)
 }
 
@@ -208,7 +208,7 @@ func (r *Robtex) executeASNAddrQuery(addr string) {
 
 	r.SetActive()
 	time.Sleep(r.RateLimit)
-	req.Netblocks = utils.UniqueAppend(req.Netblocks, r.netblocks(req.ASN)...)
+	req.Netblocks.Union(r.netblocks(req.ASN))
 	r.Bus().Publish(requests.NewASNTopic, req)
 }
 
@@ -312,14 +312,14 @@ func (r *Robtex) origin(addr string) *requests.ASNRequest {
 		ASN:         ipinfo.ASN,
 		Prefix:      ipinfo.Prefix,
 		Description: desc,
-		Netblocks:   []string{ipinfo.Prefix},
+		Netblocks:   stringset.New(ipinfo.Prefix),
 		Tag:         r.SourceType,
 		Source:      r.String(),
 	}
 }
 
-func (r *Robtex) netblocks(asn int) []string {
-	var netblocks []string
+func (r *Robtex) netblocks(asn int) stringset.Set {
+	netblocks := stringset.New()
 
 	r.SetActive()
 	url := "https://freeapi.robtex.com/asquery/" + strconv.Itoa(asn)
@@ -340,7 +340,7 @@ func (r *Robtex) netblocks(asn int) []string {
 	}
 
 	for _, net := range n.Networks {
-		netblocks = utils.UniqueAppend(netblocks, net.CIDR)
+		netblocks.Insert(net.CIDR)
 	}
 
 	if len(netblocks) == 0 {
diff --git a/services/sources/shadowserver.go b/services/sources/shadowserver.go
index f206e3f34..3e59266a5 100644
--- a/services/sources/shadowserver.go
+++ b/services/sources/shadowserver.go
@@ -17,6 +17,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -98,12 +99,12 @@ func (s *ShadowServer) executeASNQuery(asn int) {
 	}
 
 	time.Sleep(s.RateLimit)
-	req := s.origin(strings.Trim(blocks[0], "/"))
+	req := s.origin(strings.Trim(blocks.ToSlice()[0], "/"))
 	if req == nil {
 		return
 	}
 
-	req.Netblocks = utils.UniqueAppend(req.Netblocks, blocks...)
+	req.Netblocks.Union(blocks)
 	s.Bus().Publish(requests.NewASNTopic, req)
 }
 
@@ -115,7 +116,7 @@ func (s *ShadowServer) executeASNAddrQuery(addr string) {
 	}
 
 	time.Sleep(s.RateLimit)
-	req.Netblocks = utils.UniqueAppend(req.Netblocks, s.netblocks(req.ASN)...)
+	req.Netblocks.Union(s.netblocks(req.ASN))
 	s.Bus().Publish(requests.NewASNTopic, req)
 }
 
@@ -154,14 +155,14 @@ func (s *ShadowServer) origin(addr string) *requests.ASNRequest {
 		Prefix:      strings.TrimSpace(fields[1]),
 		CC:          strings.TrimSpace(fields[3]),
 		Description: strings.TrimSpace(fields[2]) + " - " + strings.TrimSpace(fields[4]),
-		Netblocks:   []string{strings.TrimSpace(fields[1])},
+		Netblocks:   stringset.New(strings.TrimSpace(fields[1])),
 		Tag:         s.SourceType,
 		Source:      s.String(),
 	}
 }
 
-func (s *ShadowServer) netblocks(asn int) []string {
-	var netblocks []string
+func (s *ShadowServer) netblocks(asn int) stringset.Set {
+	netblocks := stringset.New()
 
 	if s.addr == "" {
 		answers, err := s.Pool().Resolve(ShadowServerWhoisURL, "A", resolvers.PriorityHigh)
@@ -200,7 +201,7 @@ func (s *ShadowServer) netblocks(asn int) []string {
 		line := scanner.Text()
 
 		if err := scanner.Err(); err == nil {
-			netblocks = utils.UniqueAppend(netblocks, strings.TrimSpace(line))
+			netblocks.Insert(strings.TrimSpace(line))
 		}
 	}
 
diff --git a/services/sources/sources.go b/services/sources/sources.go
index 76dc09d91..d4d0a3394 100644
--- a/services/sources/sources.go
+++ b/services/sources/sources.go
@@ -14,6 +14,7 @@ import (
 	eb "github.com/OWASP/Amass/eventbus"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 	"github.com/PuerkitoBio/goquery"
 	"github.com/geziyor/geziyor"
@@ -103,14 +104,14 @@ func cleanName(name string) string {
 }
 
 func crawl(service services.Service, baseURL, baseDomain, subdomain, domain string) ([]string, error) {
-	var results []string
+	results := stringset.New()
 
 	maxCrawlSem.Acquire(1)
 	defer maxCrawlSem.Release(1)
 
 	re := service.Config().DomainRegex(domain)
 	if re == nil {
-		return results, fmt.Errorf("crawler error: Failed to obtain regex object for: %s", domain)
+		return results.ToSlice(), fmt.Errorf("crawler error: Failed to obtain regex object for: %s", domain)
 	}
 
 	start := fmt.Sprintf("%s/%s/%s", baseURL, strconv.Itoa(time.Now().Year()), subdomain)
@@ -128,12 +129,12 @@ func crawl(service services.Service, baseURL, baseDomain, subdomain, domain stri
 			r.HTMLDoc.Find("a").Each(func(i int, s *goquery.Selection) {
 				if href, ok := s.Attr("href"); ok {
 					if sub := re.FindString(r.JoinURL(href)); sub != "" {
-						results = utils.UniqueAppend(results, cleanName(sub))
+						results.Insert(cleanName(sub))
 					}
 				}
 			})
 		},
 	}).Start()
 
-	return results, nil
+	return results.ToSlice(), nil
 }
diff --git a/services/sources/teamcymru.go b/services/sources/teamcymru.go
index 55b5f2f8a..23723fe8b 100644
--- a/services/sources/teamcymru.go
+++ b/services/sources/teamcymru.go
@@ -15,6 +15,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -138,7 +139,7 @@ func (t *TeamCymru) origin(addr string) *requests.ASNRequest {
 		CC:             strings.TrimSpace(fields[2]),
 		Registry:       strings.TrimSpace(fields[3]),
 		AllocationDate: at,
-		Netblocks:      []string{strings.TrimSpace(fields[1])},
+		Netblocks:      stringset.New(strings.TrimSpace(fields[1])),
 		Tag:            t.SourceType,
 		Source:         t.String(),
 	}
diff --git a/services/sources/umbrella.go b/services/sources/umbrella.go
index 4cb76db72..ae4e524e2 100644
--- a/services/sources/umbrella.go
+++ b/services/sources/umbrella.go
@@ -14,6 +14,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -173,24 +174,24 @@ type rWhoisResponse struct {
 }
 
 func (u *Umbrella) collateEmails(record *whoisRecord) []string {
-	var emails []string
+	emails := stringset.New()
 
 	if u.validateScope(record.AdminContactEmail) {
-		emails = utils.UniqueAppend(emails, record.AdminContactEmail)
+		emails.InsertMany(record.AdminContactEmail)
 	}
 	if u.validateScope(record.BillingContactEmail) {
-		emails = utils.UniqueAppend(emails, record.BillingContactEmail)
+		emails.InsertMany(record.BillingContactEmail)
 	}
 	if u.validateScope(record.RegistrantEmail) {
-		emails = utils.UniqueAppend(emails, record.RegistrantEmail)
+		emails.InsertMany(record.RegistrantEmail)
 	}
 	if u.validateScope(record.TechContactEmail) {
-		emails = utils.UniqueAppend(emails, record.TechContactEmail)
+		emails.InsertMany(record.TechContactEmail)
 	}
 	if u.validateScope(record.ZoneContactEmail) {
-		emails = utils.UniqueAppend(emails, record.ZoneContactEmail)
+		emails.InsertMany(record.ZoneContactEmail)
 	}
-	return emails
+	return emails.ToSlice()
 }
 
 func (u *Umbrella) queryWhois(domain string) *whoisRecord {
@@ -217,7 +218,7 @@ func (u *Umbrella) queryWhois(domain string) *whoisRecord {
 }
 
 func (u *Umbrella) queryReverseWhois(apiURL string) []string {
-	var domains []string
+	domains := stringset.New()
 	headers := u.restHeaders()
 	var whois map[string]rWhoisResponse
 
@@ -228,7 +229,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string {
 		record, err := utils.RequestWebPage(fullAPIURL, nil, headers, "", "")
 		if err != nil {
 			u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), apiURL, err))
-			return domains
+			return domains.ToSlice()
 		}
 		err = json.Unmarshal([]byte(record), &whois)
 
@@ -237,7 +238,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string {
 			if result.TotalResults > 0 {
 				for _, domain := range result.Domains {
 					if domain.Current {
-						domains = utils.UniqueAppend(domains, domain.Domain)
+						domains.Insert(domain.Domain)
 					}
 				}
 			}
@@ -249,7 +250,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string {
 		u.SetActive()
 		time.Sleep(u.RateLimit)
 	}
-	return domains
+	return domains.ToSlice()
 }
 
 func (u *Umbrella) validateScope(input string) bool {
@@ -269,13 +270,13 @@ func (u *Umbrella) executeWhoisQuery(domain string) {
 		return
 	}
 
-	var domains []string
+	domains := stringset.New()
 	emails := u.collateEmails(whoisRecord)
 	if len(emails) > 0 {
 		emailURL := u.reverseWhoisByEmailURL(emails...)
 		for _, d := range u.queryReverseWhois(emailURL) {
 			if !u.Config().IsDomainInScope(d) {
-				domains = utils.UniqueAppend(domains, d)
+				domains.Insert(d)
 			}
 		}
 	}
@@ -290,7 +291,7 @@ func (u *Umbrella) executeWhoisQuery(domain string) {
 		nsURL := u.reverseWhoisByNSURL(nameservers...)
 		for _, d := range u.queryReverseWhois(nsURL) {
 			if !u.Config().IsDomainInScope(d) {
-				domains = utils.UniqueAppend(domains, d)
+				domains.Insert(d)
 			}
 		}
 	}
@@ -298,7 +299,7 @@ func (u *Umbrella) executeWhoisQuery(domain string) {
 	if len(domains) > 0 {
 		u.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{
 			Domain:     domain,
-			NewDomains: domains,
+			NewDomains: domains.ToSlice(),
 			Tag:        u.SourceType,
 			Source:     u.String(),
 		})
diff --git a/services/sources/urlscan.go b/services/sources/urlscan.go
index d3204f476..418c8a12f 100644
--- a/services/sources/urlscan.go
+++ b/services/sources/urlscan.go
@@ -14,6 +14,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -110,12 +111,12 @@ func (u *URLScan) executeQuery(domain string) {
 		}
 	}
 
-	var subs []string
+	subs := stringset.New()
 	for _, id := range ids {
-		subs = utils.UniqueAppend(subs, u.getSubsFromResult(id)...)
+		subs.Union(u.getSubsFromResult(id))
 	}
 
-	for _, name := range subs {
+	for name := range subs {
 		if re.MatchString(name) {
 			u.Bus().Publish(requests.NewNameTopic, &requests.DNSRequest{
 				Name:   name,
@@ -127,8 +128,8 @@ func (u *URLScan) executeQuery(domain string) {
 	}
 }
 
-func (u *URLScan) getSubsFromResult(id string) []string {
-	var subs []string
+func (u *URLScan) getSubsFromResult(id string) stringset.Set {
+	subs := stringset.New()
 
 	url := u.resultURL(id)
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
@@ -144,7 +145,7 @@ func (u *URLScan) getSubsFromResult(id string) []string {
 		} `json:"lists"`
 	}
 	if err := json.Unmarshal([]byte(page), &data); err == nil {
-		subs = utils.UniqueAppend(subs, data.Lists.Subdomains...)
+		subs.InsertMany(data.Lists.Subdomains...)
 	}
 	return subs
 }
diff --git a/services/sources/viewdns.go b/services/sources/viewdns.go
index d7c25385c..5fc89a5cb 100644
--- a/services/sources/viewdns.go
+++ b/services/sources/viewdns.go
@@ -14,6 +14,7 @@ import (
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 	"github.com/OWASP/Amass/services"
+	"github.com/OWASP/Amass/stringset"
 	"github.com/OWASP/Amass/utils"
 )
 
@@ -122,18 +123,18 @@ func (v *ViewDNS) executeWhoisQuery(domain string) {
 	re := regexp.MustCompile("<tr><td>([a-zA-Z0-9]{1}[a-zA-Z0-9-]{0,61}[a-zA-Z0-9]{1}[.]{1}[a-zA-Z0-9-]+)</td><td>")
 	subs := re.FindAllStringSubmatch(table, -1)
 
-	var matches []string
+	matches := stringset.New()
 	for _, match := range subs {
 		sub := match[1]
 		if sub != "" {
-			matches = utils.UniqueAppend(matches, strings.TrimSpace(sub))
+			matches.Insert(strings.TrimSpace(sub))
 		}
 	}
 
 	if len(matches) > 0 {
 		v.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{
 			Domain:     domain,
-			NewDomains: matches,
+			NewDomains: matches.ToSlice(),
 			Tag:        v.SourceType,
 			Source:     v.String(),
 		})
diff --git a/utils/misc.go b/utils/misc.go
index f8c7d737a..99235e086 100644
--- a/utils/misc.go
+++ b/utils/misc.go
@@ -121,11 +121,6 @@ func NewUniqueElements(orig []string, add ...string) []string {
 	return n
 }
 
-// UniqueAppend behaves like the Go append, but does not add duplicate elements.
-func UniqueAppend(orig []string, add ...string) []string {
-	return append(orig, NewUniqueElements(orig, add...)...)
-}
-
 // CopyString return a new string variable with the same value as the parameter.
 func CopyString(src string) string {
 	str := make([]byte, len(src))
diff --git a/utils/misc_test.go b/utils/misc_test.go
index 919f89da3..3a99f60c6 100644
--- a/utils/misc_test.go
+++ b/utils/misc_test.go
@@ -54,28 +54,6 @@ func TestNewUniqueElements(t *testing.T) {
 	}
 }
 
-func TestUniqueAppend(t *testing.T) {
-	tests := []struct {
-		name     string
-		orig     []string
-		event    string
-		expected []string
-	}{
-		{"Test 1: Duplicate elements", []string{"sub1.owasp.org", "sub2.owasp.org"}, "sub2.owasp.org", []string{"sub1.owasp.org", "sub2.owasp.org"}},
-		{"Test 2: New element", []string{"sub1.owasp.org", "sub2.owasp.org", "sub3.owasp.org"}, "sub4.owasp.org", []string{"sub1.owasp.org", "sub2.owasp.org", "sub3.owasp.org", "sub4.owasp.org"}},
-	}
-	for _, tt := range tests {
-		s := UniqueAppend(tt.orig, tt.event)
-		i := 0
-		for _, x := range s {
-			if x != tt.expected[i] {
-				t.Errorf("Error in %s, got %s, expected %s.", tt.name, x, tt.expected[i])
-			}
-			i++
-		}
-	}
-}
-
 func TestRemoveAsteriskLabel(t *testing.T) {
 	tests := []struct {
 		name     string
diff --git a/utils/parse.go b/utils/parse.go
index 257012739..16ac07a3b 100644
--- a/utils/parse.go
+++ b/utils/parse.go
@@ -8,8 +8,15 @@ import (
 	"net"
 	"strconv"
 	"strings"
+
+	"github.com/OWASP/Amass/stringset"
 )
 
+// ParseSet implements the flag.Value interface.
+type ParseSet struct {
+	s *stringset.Set
+}
+
 // ParseStrings implements the flag.Value interface.
 type ParseStrings []string
 
@@ -22,6 +29,29 @@ type ParseIPs []net.IP
 // ParseCIDRs implements the flag.Value interface.
 type ParseCIDRs []*net.IPNet
 
+func (p *ParseSet) String() string {
+	if p == nil {
+		return ""
+	}
+	return strings.Join(p.s.ToSlice(), ",")
+}
+
+// Set implements the flag.Value interface.
+func (p *ParseSet) Set(s string) error {
+	if p.s == nil {
+		*p.s = stringset.New()
+	}
+	if s == "" {
+		return fmt.Errorf("String parsing failed")
+	}
+
+	str := strings.Split(s, ",")
+	for _, s := range str {
+		p.s.Insert(strings.TrimSpace(s))
+	}
+	return nil
+}
+
 func (p *ParseStrings) String() string {
 	if p == nil {
 		return ""
diff --git a/utils/web.go b/utils/web.go
index 264d06904..7441b69fa 100644
--- a/utils/web.go
+++ b/utils/web.go
@@ -18,6 +18,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/OWASP/Amass/stringset"
 	"github.com/caffix/cloudflare-roundtripper/cfrt"
 )
 
@@ -174,18 +175,18 @@ func namesFromCert(cert *x509.Certificate) []string {
 		}
 	}
 
-	var subdomains []string
+	subdomains := stringset.New()
 	// Add the subject common name to the list of subdomain names
 	commonName := RemoveAsteriskLabel(cn)
 	if commonName != "" {
-		subdomains = append(subdomains, commonName)
+		subdomains.Insert(commonName)
 	}
 	// Add the cert DNS names to the list of subdomain names
 	for _, name := range cert.DNSNames {
 		n := RemoveAsteriskLabel(name)
 		if n != "" {
-			subdomains = UniqueAppend(subdomains, n)
+			subdomains.Insert(n)
 		}
 	}
-	return subdomains
+	return subdomains.ToSlice()
 }

From 1dee50f50752dc2ff9542c75c766947418221e06 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Tue, 30 Jul 2019 09:52:35 -0400
Subject: [PATCH 20/33] Include set implementation

---
 stringset/set.go      | 106 ++++++++++++++++++++++++++++++++++++++++++
 stringset/set_test.go |  91 ++++++++++++++++++++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 stringset/set.go
 create mode 100644 stringset/set_test.go

diff --git a/stringset/set.go b/stringset/set.go
new file mode 100644
index 000000000..57be257f0
--- /dev/null
+++ b/stringset/set.go
@@ -0,0 +1,106 @@
+package stringset
+
+import (
+	"fmt"
+	"strings"
+)
+
+type (
+	Set map[string]nothing
+
+	nothing struct{}
+)
+
+func New(initial ...string) Set {
+	s := Set{}
+
+	for _, v := range initial {
+		s.Insert(v)
+	}
+
+	return s
+}
+
+func (s Set) Has(element string) bool {
+	_, exists := s[element]
+	return exists
+}
+
+func (s Set) Insert(element string) {
+	s[strings.ToLower(element)] = nothing{}
+}
+
+func (s Set) InsertMany(elements ...string) {
+	for _, i := range elements {
+		s.Insert(i)
+	}
+}
+
+func (s Set) Remove(element string) {
+	delete(s, element)
+}
+
+func (s Set) ToSlice() []string {
+	var i uint64
+
+	k := make([]string, len(s))
+
+	for key := range s {
+		k[i] = key
+		i++
+	}
+
+	return k
+}
+
+func (s Set) Union(other Set) {
+	for k := range other {
+		s.Insert(k)
+	}
+}
+
+func (s Set) Len() int {
+	return len(s)
+}
+
+func (s Set) Subtract(other Set) {
+	for item := range other {
+		if s.Has(item) {
+			s.Remove(item)
+		}
+	}
+}
+
+func (s Set) Intersect(other Set) {
+	intersect := New()
+
+	for item := range other {
+		if s.Has(item) {
+			intersect.Insert(item)
+		}
+	}
+
+	for item := range s {
+		if !intersect.Has(item) {
+			s.Remove(item)
+		}
+	}
+}
+
+// Set implements the flag.Value interface.
+func (s *Set) String() string {
+	return strings.Join(s.ToSlice(), ",")
+}
+
+// Set implements the flag.Value interface.
+func (s *Set) Set(input string) error {
+	if input == "" {
+		return fmt.Errorf("String parsing failed")
+	}
+
+	items := strings.Split(input, ",")
+	for _, item := range items {
+		s.Insert(strings.TrimSpace(item))
+	}
+	return nil
+}
diff --git a/stringset/set_test.go b/stringset/set_test.go
new file mode 100644
index 000000000..d5d7a9fdf
--- /dev/null
+++ b/stringset/set_test.go
@@ -0,0 +1,91 @@
+package stringset
+
+import (
+	"testing"
+)
+
+func TestSetHas(t *testing.T) {
+	set := New("test1", "test2", "test3", "test1", "test2")
+	if !set.Has("test1") {
+		t.Errorf("Set missing expected value")
+	}
+}
+
+func TestSetInsert(t *testing.T) {
+	expected := 3
+	set := New()
+	set.Insert("test1")
+	set.Insert("test2")
+	set.Insert("test3")
+	set.Insert("test3")
+	set.Insert("test2")
+	set.Insert("test1")
+	if len(set) != expected {
+		t.Errorf("Got %d, expected %d", len(set), expected)
+	}
+}
+
+func TestSetInsertMany(t *testing.T) {
+	expected := 3
+	set := New()
+	set.InsertMany("test1", "test2", "test3", "test1", "test2")
+	if len(set) != expected {
+		t.Errorf("Got %d, expected %d", len(set), expected)
+	}
+}
+
+func TestSetRemove(t *testing.T) {
+	expected := 2
+	set := New("test1", "test2", "test3", "test1", "test2")
+	set.Remove("test1")
+	if len(set) != expected {
+		t.Errorf("Got %d, expected %d", len(set), expected)
+	}
+}
+
+func TestSetToSlice(t *testing.T) {
+	expected := 3
+	set := New("test1", "test2", "test3", "test1", "test2")
+	slice := set.ToSlice()
+	if len(slice) != expected {
+		t.Errorf("Got %d, expected %d", len(set), expected)
+	}
+}
+
+func TestSetLen(t *testing.T) {
+	expected := 3
+	set := New("test1", "test2", "test3", "test1", "test2")
+	if len(set) != expected {
+		t.Errorf("Got %d, expected %d", len(set), expected)
+	}
+}
+
+func TestSetUnion(t *testing.T) {
+	expected := 6
+	set1 := New("test1", "test2", "test3", "test6")
+	set2 := New("test1", "test2", "test3", "test4", "test5")
+	set1.Union(set2)
+	if len(set1) != expected {
+		t.Errorf("Got %d, expected %d", len(set1), expected)
+	}
+}
+
+func TestSetIntersect(t *testing.T) {
+	expected := 3
+	set1 := New("test1", "test2", "test3", "test6")
+	set2 := New("test1", "test2", "test3", "test4", "test5")
+	set1.Intersect(set2)
+	if len(set1) != expected {
+		t.Errorf("Got %d, expected %d", len(set1), expected)
+	}
+}
+
+func TestSetSubtract(t *testing.T) {
+	expected := 1
+	set1 := New("test1", "test2", "test3", "test6")
+	set2 := New("test1", "test2", "test3", "test4", "test5")
+	set1.Subtract(set2)
+	if len(set1) != expected {
+		t.Errorf("Got %d, expected %d", len(set1), expected)
+	}
+}

From 6a4a1bdc9d1854bc537cd21420de15d83aefc2c6 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Tue, 30 Jul 2019 09:53:25 -0400
Subject: [PATCH 21/33] Remove unneeded ParseSet type. Set implements
 flag.Value directly

---
 utils/parse.go | 28 ----------------------------
 1 file changed, 28 deletions(-)

diff --git a/utils/parse.go b/utils/parse.go
index 16ac07a3b..ed4de8858 100644
--- a/utils/parse.go
+++ b/utils/parse.go
@@ -12,11 +12,6 @@ import (
 	"github.com/OWASP/Amass/stringset"
 )
 
-// ParseSet implements the flag.Value interface.
-type ParseSet struct {
-	s *stringset.Set
-}
-
 // ParseStrings implements the flag.Value interface.
 type ParseStrings []string
 
@@ -29,29 +24,6 @@ type ParseIPs []net.IP
 // ParseCIDRs implements the flag.Value interface.
 type ParseCIDRs []*net.IPNet
 
-func (p *ParseSet) String() string {
-	if p == nil {
-		return ""
-	}
-	return strings.Join(p.s.ToSlice(), ",")
-}
-
-// Set implements the flag.Value interface.
-func (p *ParseSet) Set(s string) error {
-	if p.s == nil {
-		*p.s = stringset.New()
-	}
-	if s == "" {
-		return fmt.Errorf("String parsing failed")
-	}
-
-	str := strings.Split(s, ",")
-	for _, s := range str {
-		p.s.Insert(strings.TrimSpace(s))
-	}
-	return nil
-}
-
 func (p *ParseStrings) String() string {
 	if p == nil {
 		return ""

From e289f4e15739b9006b8d382f4acd2d1d6dcd9e13 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Tue, 30 Jul 2019 11:12:34 -0400
Subject: [PATCH 22/33] Change lists in Config back to []string

---
 cmd/amass/enum.go           | 10 +++----
 cmd/amass/intel.go          |  2 +-
 config/config.go            | 55 ++++++++++++++++++-------------------
 enum/enum.go                |  2 +-
 intel/intel.go              |  2 +-
 services/alteration.go      |  5 ++--
 services/alteration_test.go |  2 +-
 services/brute.go           |  2 +-
 stringset/set.go            |  4 +++
 utils/parse.go              |  2 --
 10 files changed, 42 insertions(+), 44 deletions(-)

diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go
index 7b382e3bb..70468a544 100644
--- a/cmd/amass/enum.go
+++ b/cmd/amass/enum.go
@@ -505,7 +505,7 @@ func processEnumInputFiles(args *enumArgs) error {
 		for _, f := range args.Filepaths.Resolvers {
 			list, err := config.GetListFromFile(f)
 			if err != nil {
-				return fmt.Errorf("Failed to parse the resolver file: %v", err)
+				return fmt.Errorf("Failed to parse the esolver file: %v", err)
 			}
 
 			args.Resolvers.InsertMany(list...)
@@ -535,10 +535,10 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
 		e.Config.MaxDNSQueries = args.MaxDNSQueries
 	}
 	if len(args.BruteWordList) > 0 {
-		e.Config.Wordlist = args.BruteWordList
+		e.Config.Wordlist = args.BruteWordList.ToSlice()
 	}
 	if len(args.AltWordList) > 0 {
-		e.Config.AltWordlist = args.AltWordList
+		e.Config.AltWordlist = args.AltWordList.ToSlice()
 	}
 	if len(args.Names) > 0 {
 		e.ProvidedNames = args.Names.ToSlice()
@@ -565,12 +565,12 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
 		e.Config.Passive = true
 	}
 	if len(args.Blacklist) > 0 {
-		e.Config.Blacklist = args.Blacklist
+		e.Config.Blacklist = args.Blacklist.ToSlice()
 	}
 
 	disabled := compileDisabledSources(e.GetAllSourceNames(), args.Included, args.Excluded)
 	if len(disabled) > 0 {
-		e.Config.DisabledDataSources = disabled
+		e.Config.DisabledDataSources = disabled.ToSlice()
 	}
 
 	// Attempt to add the provided domains to the configuration
diff --git a/cmd/amass/intel.go b/cmd/amass/intel.go
index 317df9abd..9bc15ed32 100644
--- a/cmd/amass/intel.go
+++ b/cmd/amass/intel.go
@@ -383,7 +383,7 @@ func updateIntelConfiguration(ic *intel.Collection, args *intelArgs) error {
 
 	disabled := compileDisabledSources(GetAllSourceNames(), args.Included, args.Excluded)
 	if len(disabled) > 0 {
-		ic.Config.DisabledDataSources = disabled
+		ic.Config.DisabledDataSources = disabled.ToSlice()
 	}
 	// Attempt to add the provided domains to the configuration
 	ic.Config.AddDomains(args.Domains.ToSlice())
diff --git a/config/config.go b/config/config.go
index 4a87ca9f6..5e07f7640 100644
--- a/config/config.go
+++ b/config/config.go
@@ -75,7 +75,7 @@ type Config struct {
 	Ports []int
 
 	// The list of words to use when generating names
-	Wordlist stringset.Set
+	Wordlist []string
 
 	// Will the enumeration including brute forcing techniques
 	BruteForcing bool
@@ -94,7 +94,7 @@ type Config struct {
 	AddNumbers     bool
 	MinForWordFlip int
 	EditDistance   int
-	AltWordlist    stringset.Set
+	AltWordlist    []string
 
 	// Only access the data sources for names and return results?
 	Passive bool
@@ -106,13 +106,13 @@ type Config struct {
 	IncludeUnresolvable bool `ini:"include_unresolvable"`
 
 	// A blacklist of subdomain names that will not be investigated
-	Blacklist stringset.Set
+	Blacklist []string
 
 	// A list of data sources that should not be utilized
-	DisabledDataSources stringset.Set
+	DisabledDataSources []string
 
 	// The root domain names that the enumeration will target
-	domains stringset.Set
+	domains []string
 
 	// The regular expressions for the root domains added to the enumeration
 	regexps map[string]*regexp.Regexp
@@ -136,11 +136,6 @@ func New() *Config {
 }
 
 func (c *Config) Init() {
-	c.AltWordlist = stringset.New()
-	c.Wordlist = stringset.New()
-	c.domains = stringset.New()
-	c.Blacklist = stringset.New()
-	c.DisabledDataSources = stringset.New()
 	c.regexps = make(map[string]*regexp.Regexp)
 	c.apikeys = make(map[string]*APIKey)
 }
@@ -178,19 +173,15 @@ func (c *Config) CheckSettings() error {
 	}
 	c.SemMaxDNSQueries = utils.NewSimpleSemaphore(c.MaxDNSQueries)
 
-	wordlist := c.Wordlist.ToSlice()
-	wordlist, err = utils.ExpandMaskWordlist(wordlist)
+	c.Wordlist, err = utils.ExpandMaskWordlist(c.Wordlist)
 	if err != nil {
 		return err
 	}
-	c.Wordlist = stringset.New(wordlist...)
 
-	altWordlist := c.AltWordlist.ToSlice()
-	altWordlist, err = utils.ExpandMaskWordlist(altWordlist)
+	c.AltWordlist, err = utils.ExpandMaskWordlist(c.AltWordlist)
 	if err != nil {
 		return err
 	}
-	c.AltWordlist = stringset.New(altWordlist...)
 
 	return err
 }
@@ -238,8 +229,10 @@ func (c *Config) AddDomain(domain string) {
 	c.regexps[d] = utils.SubdomainRegex(d)
 	if c.regexps[d] != nil {
 		// Add the domain string to the list
-		c.domains.Insert(d)
+		c.domains = append(c.domains, d)
 	}
+
+	c.domains = stringset.Deduplicate(c.domains)
 }
 
 // Domains returns the list of domain names currently in the configuration.
@@ -247,7 +240,7 @@ func (c *Config) Domains() []string {
 	c.Lock()
 	defer c.Unlock()
 
-	return c.domains.ToSlice()
+	return c.domains
 }
 
 // IsDomainInScope returns true if the DNS name in the parameter ends with a domain in the config list.
@@ -307,7 +300,7 @@ func (c *Config) Blacklisted(name string) bool {
 	var resp bool
 
 	n := strings.TrimSpace(name)
-	for bl := range c.Blacklist {
+	for _, bl := range c.Blacklist {
 		if match := strings.HasSuffix(n, bl); match {
 			resp = true
 			break
@@ -403,9 +396,11 @@ func (c *Config) loadBruteForceSettings(cfg *ini.File) error {
 			if err != nil {
 				return fmt.Errorf("Unable to load the file in the bruteforce wordlist_file setting: %s: %v", wordlist, err)
 			}
-			c.Wordlist.InsertMany(list...)
+			c.Wordlist = append(c.Wordlist, list...)
 		}
 	}
+
+	c.Wordlist = stringset.Deduplicate(c.Wordlist)
 	return nil
 }
 
@@ -433,9 +428,11 @@ func (c *Config) loadAlterationSettings(cfg *ini.File) error {
 			if err != nil {
 				return fmt.Errorf("Unable to load the file in the alterations wordlist_file setting: %s: %v", wordlist, err)
 			}
-			c.AltWordlist.InsertMany(list...)
+			c.AltWordlist = append(c.AltWordlist, list...)
 		}
 	}
+
+	c.AltWordlist = stringset.Deduplicate(c.AltWordlist)
 	return nil
 }
 
@@ -470,11 +467,11 @@ func (c *Config) LoadSettings(path string) error {
 	}
 	// Load up all the blacklisted subdomain names
 	if blacklisted, err := cfg.GetSection("blacklisted"); err == nil {
-		c.Blacklist.InsertMany(blacklisted.Key("subdomain").ValueWithShadows()...)
+		c.Blacklist = stringset.Deduplicate(blacklisted.Key("subdomain").ValueWithShadows())
 	}
 	// Load up all the disabled data source names
 	if disabled, err := cfg.GetSection("disabled_data_sources"); err == nil {
-		c.DisabledDataSources.InsertMany(disabled.Key("data_source").ValueWithShadows()...)
+		c.DisabledDataSources = stringset.Deduplicate(disabled.Key("data_source").ValueWithShadows())
 	}
 	// Load up all the Gremlin Server settings
 	if gremlin, err := cfg.GetSection("gremlin"); err == nil {
@@ -617,10 +614,10 @@ func GetListFromFile(path string) ([]string, error) {
 	}
 
 	s, err := getWordList(reader)
-	return s.ToSlice(), err
+	return s, err
 }
 
-func getWordlistByURL(url string) (stringset.Set, error) {
+func getWordlistByURL(url string) ([]string, error) {
 	page, err := utils.RequestWebPage(url, nil, nil, "", "")
 	if err != nil {
 		return nil, fmt.Errorf("Failed to obtain the wordlist at %s: %v", url, err)
@@ -628,18 +625,18 @@ func getWordlistByURL(url string) (stringset.Set, error) {
 	return getWordList(strings.NewReader(page))
 }
 
-func getWordList(reader io.Reader) (stringset.Set, error) {
-	words := stringset.New()
+func getWordList(reader io.Reader) ([]string, error) {
+	var words []string
 
 	scanner := bufio.NewScanner(reader)
 	for scanner.Scan() {
 		// Get the next word in the list
 		w := strings.TrimSpace(scanner.Text())
 		if err := scanner.Err(); err == nil && w != "" {
-			words.Insert(w)
+			words = append(words, w)
 		}
 	}
-	return words, nil
+	return stringset.Deduplicate(words), nil
 }
 
 func uniqueIntAppend(s []int, e string) []int {
diff --git a/enum/enum.go b/enum/enum.go
index 910e59b3c..225809c4c 100644
--- a/enum/enum.go
+++ b/enum/enum.go
@@ -464,7 +464,7 @@ func ExcludeDisabledDataSources(srvs []services.Service, cfg *config.Config) []s
 	for _, s := range srvs {
 		include := true
 
-		for disabled := range cfg.DisabledDataSources {
+		for _, disabled := range cfg.DisabledDataSources {
 			if strings.EqualFold(disabled, s.String()) {
 				include = false
 				break
diff --git a/intel/intel.go b/intel/intel.go
index 397b4241c..2d5f75e04 100644
--- a/intel/intel.go
+++ b/intel/intel.go
@@ -380,7 +380,7 @@ func ExcludeDisabledDataSources(srvs []services.Service, cfg *config.Config) []s
 	for _, s := range srvs {
 		include := true
 
-		for disabled := range cfg.DisabledDataSources {
+		for _, disabled := range cfg.DisabledDataSources {
 			if strings.EqualFold(disabled, s.String()) {
 				include = false
 				break
diff --git a/services/alteration.go b/services/alteration.go
index b95cc8a3a..b1e9a4780 100644
--- a/services/alteration.go
+++ b/services/alteration.go
@@ -60,11 +60,10 @@ type AlterationService struct {
 
 // NewAlterationService returns he object initialized, but not yet started.
 func NewAlterationService(cfg *config.Config, bus *eb.EventBus, pool *resolvers.ResolverPool) *AlterationService {
-	seed := cfg.AltWordlist.ToSlice()
 	as := &AlterationService{
 		filter:   utils.NewStringFilter(),
-		prefixes: newAlterationCache(seed),
-		suffixes: newAlterationCache(seed),
+		prefixes: newAlterationCache(cfg.AltWordlist),
+		suffixes: newAlterationCache(cfg.AltWordlist),
 	}
 
 	as.BaseService = *NewBaseService(as, "Alterations", cfg, bus, pool)
diff --git a/services/alteration_test.go b/services/alteration_test.go
index b6b68e33c..1e0547cff 100644
--- a/services/alteration_test.go
+++ b/services/alteration_test.go
@@ -36,7 +36,7 @@ func setupConfig(domain string) *config.Config {
 	cfg.AddNumbers = true
 	cfg.MinForWordFlip = 0
 	cfg.EditDistance = 1
-	cfg.AltWordlist.InsertMany("prod", "dev")
+	cfg.AltWordlist = []string{"prod", "dev"}
 	cfg.AddDomain(domain)
 	buf := new(strings.Builder)
 	cfg.Log = log.New(buf, "", log.Lmicroseconds)
diff --git a/services/brute.go b/services/brute.go
index 7532a5876..54be724f8 100644
--- a/services/brute.go
+++ b/services/brute.go
@@ -145,7 +145,7 @@ func (bfs *BruteForceService) performBruteForcing(subdomain, domain string) {
 		bfs.Pool().GetWildcardType(req) == resolvers.WildcardTypeDynamic {
 		return
 	}
-	wordlist := bfs.Config().Wordlist.ToSlice()
+	wordlist := bfs.Config().Wordlist
 
 	bfs.totalLock.Lock()
 	bfs.totalNames += len(wordlist)
diff --git a/stringset/set.go b/stringset/set.go
index 57be257f0..d296c17cf 100644
--- a/stringset/set.go
+++ b/stringset/set.go
@@ -11,6 +11,10 @@ type (
 	nothing struct{}
 )
 
+func Deduplicate(input []string) []string {
+	return New(input...).ToSlice()
+}
+
 func New(initial ...string) Set {
 	s := Set{}
 
diff --git a/utils/parse.go b/utils/parse.go
index ed4de8858..257012739 100644
--- a/utils/parse.go
+++ b/utils/parse.go
@@ -8,8 +8,6 @@ import (
 	"net"
 	"strconv"
 	"strings"
-
-	"github.com/OWASP/Amass/stringset"
 )
 
 // ParseStrings implements the flag.Value interface.

From 0bfab8c58433e053b47299b6c37c81f8c6a7d4bf Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Tue, 30 Jul 2019 11:34:44 -0400
Subject: [PATCH 23/33] Be more go and less java with naming conventions

---
 cmd/amass/db.go                  |  4 ++--
 cmd/amass/enum.go                | 14 +++++++-------
 cmd/amass/intel.go               |  6 +++---
 cmd/amass/track.go               |  6 +++---
 cmd/amass/viz.go                 |  4 ++--
 graph/graph.go                   |  6 +++---
 services/sources/alienvault.go   | 10 +++++-----
 services/sources/networksdb.go   |  4 ++--
 services/sources/robtex.go       |  2 +-
 services/sources/shadowserver.go |  2 +-
 services/sources/sources.go      |  4 ++--
 services/sources/umbrella.go     |  8 ++++----
 services/sources/viewdns.go      |  2 +-
 stringset/set.go                 |  6 +++---
 stringset/set_test.go            |  4 ++--
 utils/web.go                     |  2 +-
 16 files changed, 42 insertions(+), 42 deletions(-)

diff --git a/cmd/amass/db.go b/cmd/amass/db.go
index 1405076b5..d9c1c80f8 100644
--- a/cmd/amass/db.go
+++ b/cmd/amass/db.go
@@ -176,7 +176,7 @@ func inputDataOperations(args *dbArgs, db graph.DataHandler) error {
 }
 
 func listEnumerations(args *dbArgs, db graph.DataHandler) {
-	domains := args.Domains.ToSlice()
+	domains := args.Domains.Slice()
 	enums := enumIDs(domains, db)
 	if len(enums) == 0 {
 		r.Fprintln(color.Error, "No enumerations found within the provided scope")
@@ -202,7 +202,7 @@ func listEnumerations(args *dbArgs, db graph.DataHandler) {
 }
 
 func showEnumeration(args *dbArgs, db graph.DataHandler) {
-	domains := args.Domains.ToSlice()
+	domains := args.Domains.Slice()
 	var total int
 	tags := make(map[string]int)
 	asns := make(map[int]*utils.ASNSummaryData)
diff --git a/cmd/amass/enum.go b/cmd/amass/enum.go
index 70468a544..9eb05ef4d 100644
--- a/cmd/amass/enum.go
+++ b/cmd/amass/enum.go
@@ -233,7 +233,7 @@ func runEnumCommand(clArgs []string) {
 	}
 
 	if len(args.Resolvers) > 0 {
-		if err := e.Pool.SetResolvers(args.Resolvers.ToSlice()); err != nil {
+		if err := e.Pool.SetResolvers(args.Resolvers.Slice()); err != nil {
 			r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err)
 			os.Exit(1)
 		}
@@ -535,13 +535,13 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
 		e.Config.MaxDNSQueries = args.MaxDNSQueries
 	}
 	if len(args.BruteWordList) > 0 {
-		e.Config.Wordlist = args.BruteWordList.ToSlice()
+		e.Config.Wordlist = args.BruteWordList.Slice()
 	}
 	if len(args.AltWordList) > 0 {
-		e.Config.AltWordlist = args.AltWordList.ToSlice()
+		e.Config.AltWordlist = args.AltWordList.Slice()
 	}
 	if len(args.Names) > 0 {
-		e.ProvidedNames = args.Names.ToSlice()
+		e.ProvidedNames = args.Names.Slice()
 	}
 	if args.Options.BruteForcing {
 		e.Config.BruteForcing = true
@@ -565,16 +565,16 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
 		e.Config.Passive = true
 	}
 	if len(args.Blacklist) > 0 {
-		e.Config.Blacklist = args.Blacklist.ToSlice()
+		e.Config.Blacklist = args.Blacklist.Slice()
 	}
 
 	disabled := compileDisabledSources(e.GetAllSourceNames(), args.Included, args.Excluded)
 	if len(disabled) > 0 {
-		e.Config.DisabledDataSources = disabled.ToSlice()
+		e.Config.DisabledDataSources = disabled.Slice()
 	}
 
 	// Attempt to add the provided domains to the configuration
-	e.Config.AddDomains(args.Domains.ToSlice())
+	e.Config.AddDomains(args.Domains.Slice())
 	if len(e.Config.Domains()) == 0 {
 		return errors.New("No root domain names were provided")
 	}
diff --git a/cmd/amass/intel.go b/cmd/amass/intel.go
index 9bc15ed32..218005ac2 100644
--- a/cmd/amass/intel.go
+++ b/cmd/amass/intel.go
@@ -192,7 +192,7 @@ func runIntelCommand(clArgs []string) {
 	}
 
 	if len(args.Resolvers) > 0 {
-		if err := ic.Pool.SetResolvers(args.Resolvers.ToSlice()); err != nil {
+		if err := ic.Pool.SetResolvers(args.Resolvers.Slice()); err != nil {
 			r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err)
 			os.Exit(1)
 		}
@@ -383,9 +383,9 @@ func updateIntelConfiguration(ic *intel.Collection, args *intelArgs) error {
 
 	disabled := compileDisabledSources(GetAllSourceNames(), args.Included, args.Excluded)
 	if len(disabled) > 0 {
-		ic.Config.DisabledDataSources = disabled.ToSlice()
+		ic.Config.DisabledDataSources = disabled.Slice()
 	}
 	// Attempt to add the provided domains to the configuration
-	ic.Config.AddDomains(args.Domains.ToSlice())
+	ic.Config.AddDomains(args.Domains.Slice())
 	return nil
 }
diff --git a/cmd/amass/track.go b/cmd/amass/track.go
index 928e52111..8ea2a36d9 100644
--- a/cmd/amass/track.go
+++ b/cmd/amass/track.go
@@ -129,7 +129,7 @@ func runTrackCommand(clArgs []string) {
 	defer db.Close()
 
 	// Obtain the enumerations that include the provided domain(s)
-	enums := enumIDs(args.Domains.ToSlice(), db)
+	enums := enumIDs(args.Domains.Slice(), db)
 
 	// There needs to be at least two enumerations to proceed
 	if len(enums) < 2 {
@@ -164,10 +164,10 @@ func runTrackCommand(clArgs []string) {
 	latest = latest[:end]
 
 	if args.Options.History {
-		completeHistoryOutput(args.Domains.ToSlice(), enums, earliest, latest, db)
+		completeHistoryOutput(args.Domains.Slice(), enums, earliest, latest, db)
 		return
 	}
-	cumulativeOutput(args.Domains.ToSlice(), enums, earliest, latest, db)
+	cumulativeOutput(args.Domains.Slice(), enums, earliest, latest, db)
 }
 
 func cumulativeOutput(domains []string, enums []string, ea, la []time.Time, db graph.DataHandler) {
diff --git a/cmd/amass/viz.go b/cmd/amass/viz.go
index cc751bc06..f9ff060f6 100644
--- a/cmd/amass/viz.go
+++ b/cmd/amass/viz.go
@@ -148,10 +148,10 @@ func runVizCommand(clArgs []string) {
 		defer db.Close()
 
 		if args.Enum > 0 {
-			uuid = enumIndexToID(args.Enum, args.Domains.ToSlice(), db)
+			uuid = enumIndexToID(args.Enum, args.Domains.Slice(), db)
 		} else {
 			// Get the UUID for the most recent enumeration
-			uuid = mostRecentEnumID(args.Domains.ToSlice(), db)
+			uuid = mostRecentEnumID(args.Domains.Slice(), db)
 		}
 	}
 	if uuid == "" {
diff --git a/graph/graph.go b/graph/graph.go
index 07d433fa7..804129761 100644
--- a/graph/graph.go
+++ b/graph/graph.go
@@ -456,7 +456,7 @@ func (g *Graph) EnumerationList() []string {
 			ids.Insert(label)
 		}
 	}
-	return ids.ToSlice()
+	return ids.Slice()
 }
 
 // EnumerationDomains returns the domains that were involved in the provided enumeration.
@@ -480,7 +480,7 @@ func (g *Graph) EnumerationDomains(uuid string) []string {
 			domains.Insert(domain)
 		}
 	}
-	return domains.ToSlice()
+	return domains.Slice()
 }
 
 // EnumerationDateRange returns the date range associated with the provided enumeration UUID.
@@ -600,7 +600,7 @@ func (g *Graph) getCNAMEs(sub, uuid string) []string {
 		cname = quad.String(target)
 		names.Insert(target)
 	}
-	return names.ToSlice()
+	return names.Slice()
 }
 
 func (g *Graph) buildOutput(sub, uuid string) *requests.Output {
diff --git a/services/sources/alienvault.go b/services/sources/alienvault.go
index 5bc451dad..3cbbac9ce 100644
--- a/services/sources/alienvault.go
+++ b/services/sources/alienvault.go
@@ -254,7 +254,7 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 	page, err := utils.RequestWebPage(u, nil, a.getHeaders(), "", "")
 	if err != nil {
 		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
-		return emails.ToSlice()
+		return emails.Slice()
 	}
 
 	var m struct {
@@ -267,12 +267,12 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 	}
 	if err := json.Unmarshal([]byte(page), &m); err != nil {
 		a.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", a.String(), u, err))
-		return emails.ToSlice()
+		return emails.Slice()
 	} else if m.Count == 0 {
 		a.Bus().Publish(requests.LogTopic,
 			fmt.Sprintf("%s: %s: The query returned zero results", a.String(), u),
 		)
-		return emails.ToSlice()
+		return emails.Slice()
 	}
 
 	for _, row := range m.Data {
@@ -291,7 +291,7 @@ func (a *AlienVault) queryWhoisForEmails(domain string) []string {
 			}
 		}
 	}
-	return emails.ToSlice()
+	return emails.Slice()
 }
 
 func (a *AlienVault) executeWhoisQuery(domain string) {
@@ -334,7 +334,7 @@ func (a *AlienVault) executeWhoisQuery(domain string) {
 
 	a.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{
 		Domain:     domain,
-		NewDomains: newDomains.ToSlice(),
+		NewDomains: newDomains.Slice(),
 		Tag:        a.SourceType,
 		Source:     a.String(),
 	})
diff --git a/services/sources/networksdb.go b/services/sources/networksdb.go
index 5c9d3bbae..f12059042 100644
--- a/services/sources/networksdb.go
+++ b/services/sources/networksdb.go
@@ -213,7 +213,7 @@ func (n *NetworksDB) executeASNQuery(asn int, addr string, netblocks stringset.S
 		}
 	}
 	if prefix == "" && len(netblocks) > 0 {
-		prefix = netblocks.ToSlice()[0] // TODO order may matter here :shrug:
+		prefix = netblocks.Slice()[0] // TODO order may matter here :shrug:
 	}
 
 	n.Bus().Publish(requests.NewASNTopic, &requests.ASNRequest{
@@ -304,7 +304,7 @@ func (n *NetworksDB) executeAPIASNQuery(asn int, addr string, netblocks stringse
 		}
 	}
 	if prefix == "" {
-		prefix = netblocks.ToSlice()[0]
+		prefix = netblocks.Slice()[0]
 	}
 
 	time.Sleep(n.RateLimit)
diff --git a/services/sources/robtex.go b/services/sources/robtex.go
index a074d3b45..b88641119 100644
--- a/services/sources/robtex.go
+++ b/services/sources/robtex.go
@@ -183,7 +183,7 @@ func (r *Robtex) executeASNQuery(asn int) {
 		return
 	}
 
-	_, ipnet, err := net.ParseCIDR(blocks.ToSlice()[0])
+	_, ipnet, err := net.ParseCIDR(blocks.Slice()[0])
 	if err != nil {
 		return
 	}
diff --git a/services/sources/shadowserver.go b/services/sources/shadowserver.go
index 3e59266a5..7a654c41c 100644
--- a/services/sources/shadowserver.go
+++ b/services/sources/shadowserver.go
@@ -99,7 +99,7 @@ func (s *ShadowServer) executeASNQuery(asn int) {
 	}
 
 	time.Sleep(s.RateLimit)
-	req := s.origin(strings.Trim(blocks.ToSlice()[0], "/"))
+	req := s.origin(strings.Trim(blocks.Slice()[0], "/"))
 	if req == nil {
 		return
 	}
diff --git a/services/sources/sources.go b/services/sources/sources.go
index d4d0a3394..6dd8d250c 100644
--- a/services/sources/sources.go
+++ b/services/sources/sources.go
@@ -111,7 +111,7 @@ func crawl(service services.Service, baseURL, baseDomain, subdomain, domain stri
 
 	re := service.Config().DomainRegex(domain)
 	if re == nil {
-		return results.ToSlice(), fmt.Errorf("crawler error: Failed to obtain regex object for: %s", domain)
+		return results.Slice(), fmt.Errorf("crawler error: Failed to obtain regex object for: %s", domain)
 	}
 
 	start := fmt.Sprintf("%s/%s/%s", baseURL, strconv.Itoa(time.Now().Year()), subdomain)
@@ -136,5 +136,5 @@ func crawl(service services.Service, baseURL, baseDomain, subdomain, domain stri
 		},
 	}).Start()
 
-	return results.ToSlice(), nil
+	return results.Slice(), nil
 }
diff --git a/services/sources/umbrella.go b/services/sources/umbrella.go
index ae4e524e2..c33ea50df 100644
--- a/services/sources/umbrella.go
+++ b/services/sources/umbrella.go
@@ -191,7 +191,7 @@ func (u *Umbrella) collateEmails(record *whoisRecord) []string {
 	if u.validateScope(record.ZoneContactEmail) {
 		emails.InsertMany(record.ZoneContactEmail)
 	}
-	return emails.ToSlice()
+	return emails.Slice()
 }
 
 func (u *Umbrella) queryWhois(domain string) *whoisRecord {
@@ -229,7 +229,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string {
 		record, err := utils.RequestWebPage(fullAPIURL, nil, headers, "", "")
 		if err != nil {
 			u.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", u.String(), apiURL, err))
-			return domains.ToSlice()
+			return domains.Slice()
 		}
 		err = json.Unmarshal([]byte(record), &whois)
 
@@ -250,7 +250,7 @@ func (u *Umbrella) queryReverseWhois(apiURL string) []string {
 		u.SetActive()
 		time.Sleep(u.RateLimit)
 	}
-	return domains.ToSlice()
+	return domains.Slice()
 }
 
 func (u *Umbrella) validateScope(input string) bool {
@@ -299,7 +299,7 @@ func (u *Umbrella) executeWhoisQuery(domain string) {
 	if len(domains) > 0 {
 		u.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{
 			Domain:     domain,
-			NewDomains: domains.ToSlice(),
+			NewDomains: domains.Slice(),
 			Tag:        u.SourceType,
 			Source:     u.String(),
 		})
diff --git a/services/sources/viewdns.go b/services/sources/viewdns.go
index 5fc89a5cb..66a67b3b5 100644
--- a/services/sources/viewdns.go
+++ b/services/sources/viewdns.go
@@ -134,7 +134,7 @@ func (v *ViewDNS) executeWhoisQuery(domain string) {
 	if len(matches) > 0 {
 		v.Bus().Publish(requests.NewWhoisTopic, &requests.WhoisRequest{
 			Domain:     domain,
-			NewDomains: matches.ToSlice(),
+			NewDomains: matches.Slice(),
 			Tag:        v.SourceType,
 			Source:     v.String(),
 		})
diff --git a/stringset/set.go b/stringset/set.go
index d296c17cf..e038ef72c 100644
--- a/stringset/set.go
+++ b/stringset/set.go
@@ -12,7 +12,7 @@ type (
 )
 
 func Deduplicate(input []string) []string {
-	return New(input...).ToSlice()
+	return New(input...).Slice()
 }
 
 func New(initial ...string) Set {
@@ -44,7 +44,7 @@ func (s Set) Remove(element string) {
 	delete(s, element)
 }
 
-func (s Set) ToSlice() []string {
+func (s Set) Slice() []string {
 	var i uint64
 
 	k := make([]string, len(s))
@@ -93,7 +93,7 @@ func (s Set) Intersect(other Set) {
 
 // Set implements the flag.Value interface.
 func (s *Set) String() string {
-	return strings.Join(s.ToSlice(), ",")
+	return strings.Join(s.Slice(), ",")
 }
 
 // Set implements the flag.Value interface.
diff --git a/stringset/set_test.go b/stringset/set_test.go
index d5d7a9fdf..7fa1380c0 100644
--- a/stringset/set_test.go
+++ b/stringset/set_test.go
@@ -43,10 +43,10 @@ func TestSetRemove(t *testing.T) {
 	}
 }
 
-func TestSetToSlice(t *testing.T) {
+func TestSetSlice(t *testing.T) {
 	expected := 3
 	set := New("test1", "test2", "test3", "test1", "test2")
-	slice := set.ToSlice()
+	slice := set.Slice()
 	if len(slice) != expected {
 		t.Errorf("Got %d, expected %d", len(set), expected)
 	}
diff --git a/utils/web.go b/utils/web.go
index 7441b69fa..43786b288 100644
--- a/utils/web.go
+++ b/utils/web.go
@@ -188,5 +188,5 @@ func namesFromCert(cert *x509.Certificate) []string {
 			subdomains.Insert(n)
 		}
 	}
-	return subdomains.ToSlice()
+	return subdomains.Slice()
 }

From 4e3b1b8d33df70efc870b9833ecab3a90b786aa2 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Tue, 30 Jul 2019 13:23:32 -0400
Subject: [PATCH 24/33] Add some old checks back in

---
 config/config.go | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/config/config.go b/config/config.go
index 5e07f7640..f6c23db27 100644
--- a/config/config.go
+++ b/config/config.go
@@ -225,6 +225,12 @@ func (c *Config) AddDomain(domain string) {
 			return
 		}
 	}
+
+	// Check that the regular expression map has been initialized
+	if c.regexps == nil {
+		c.regexps = make(map[string]*regexp.Regexp)
+	}
+
 	// Create the regular expression for this domain
 	c.regexps[d] = utils.SubdomainRegex(d)
 	if c.regexps[d] != nil {
@@ -319,6 +325,9 @@ func (c *Config) AddAPIKey(source string, ak *APIKey) {
 		return
 	}
 
+	if c.apikeys == nil {
+		c.apikeys = make(map[string]*APIKey)
+	}
 	c.apikeys[strings.ToLower(idx)] = ak
 }
 

From 01c4d663d9e3df64a85814daccefb88dadd16c80 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Tue, 30 Jul 2019 16:34:36 -0400
Subject: [PATCH 25/33] Perform some final cleanup for this PR

---
 cmd/amass/db.go             |  2 +-
 cmd/amass/track.go          |  2 +-
 cmd/amass/viz.go            |  2 +-
 config/config.go            | 11 -----------
 enum/enum.go                |  1 -
 services/alteration_test.go |  1 -
 6 files changed, 3 insertions(+), 16 deletions(-)

diff --git a/cmd/amass/db.go b/cmd/amass/db.go
index d9c1c80f8..4f02d3046 100644
--- a/cmd/amass/db.go
+++ b/cmd/amass/db.go
@@ -94,7 +94,7 @@ func runDBCommand(clArgs []string) {
 		args.Domains.InsertMany(list...)
 	}
 
-	cfg := config.New()
+	cfg := new(config.Config)
 	// Check if a configuration file was provided, and if so, load the settings
 	if _, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {
 		if args.Filepaths.Directory == "" {
diff --git a/cmd/amass/track.go b/cmd/amass/track.go
index 8ea2a36d9..81d3112cf 100644
--- a/cmd/amass/track.go
+++ b/cmd/amass/track.go
@@ -106,7 +106,7 @@ func runTrackCommand(clArgs []string) {
 
 	rand.Seed(time.Now().UTC().UnixNano())
 
-	cfg := config.New()
+	cfg := new(config.Config)
 	// Check if a configuration file was provided, and if so, load the settings
 	if _, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {
 		if args.Filepaths.Directory == "" {
diff --git a/cmd/amass/viz.go b/cmd/amass/viz.go
index f9ff060f6..0fc610c2c 100644
--- a/cmd/amass/viz.go
+++ b/cmd/amass/viz.go
@@ -126,7 +126,7 @@ func runVizCommand(clArgs []string) {
 			defer db.Close()
 		}
 	} else {
-		cfg := config.New()
+		cfg := new(config.Config)
 		// Check if a configuration file was provided, and if so, load the settings
 		if _, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, cfg); err == nil {
 			if args.Filepaths.Directory == "" {
diff --git a/config/config.go b/config/config.go
index f6c23db27..dfa0a7273 100644
--- a/config/config.go
+++ b/config/config.go
@@ -129,17 +129,6 @@ type APIKey struct {
 	Secret   string `ini:"secret"`
 }
 
-func New() *Config {
-	c := new(Config)
-	c.Init()
-	return c
-}
-
-func (c *Config) Init() {
-	c.regexps = make(map[string]*regexp.Regexp)
-	c.apikeys = make(map[string]*APIKey)
-}
-
 // CheckSettings runs some sanity checks on the configuration options selected.
 func (c *Config) CheckSettings() error {
 	var err error
diff --git a/enum/enum.go b/enum/enum.go
index 225809c4c..0515aa2d2 100644
--- a/enum/enum.go
+++ b/enum/enum.go
@@ -81,7 +81,6 @@ func NewEnumeration() *Enumeration {
 		filter:      utils.NewStringFilter(),
 		outputQueue: new(utils.Queue),
 	}
-	e.Config.Init()
 	if e.Pool == nil {
 		return nil
 	}
diff --git a/services/alteration_test.go b/services/alteration_test.go
index 1e0547cff..24d24fcd3 100644
--- a/services/alteration_test.go
+++ b/services/alteration_test.go
@@ -28,7 +28,6 @@ var (
 
 func setupConfig(domain string) *config.Config {
 	cfg := &config.Config{}
-	cfg.Init()
 	cfg.Alterations = true
 	cfg.FlipWords = true
 	cfg.AddWords = true

From 75e2f64828bc2e1d3579afc11a7026788d5938d8 Mon Sep 17 00:00:00 2001
From: vltraheaven <vltraheaven@protonmail.com>
Date: Tue, 30 Jul 2019 23:16:02 -0700
Subject: [PATCH 26/33] Created passivetotal_test.go unit test file for
 PassiveTotal source

---
 services/sources/passivetotal_test.go | 38 +++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 services/sources/passivetotal_test.go

diff --git a/services/sources/passivetotal_test.go b/services/sources/passivetotal_test.go
new file mode 100644
index 000000000..a1dff5bf2
--- /dev/null
+++ b/services/sources/passivetotal_test.go
@@ -0,0 +1,38 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/OWASP/Amass/config"
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+)
+
+func TestPassiveTotal(t *testing.T) {
+	if *networkTest == false || *configPath == "" {
+		return
+	}
+
+	cfg := setupConfig(domainTest)
+
+	API := new(config.APIKey)
+	API = cfg.GetAPIKey("passivetotal")
+
+	if API == nil || API.Username == "" || API.Key == "" {
+		t.Errorf("API key data was not provided")
+		return
+	}
+
+	bus, out := setupEventBus(requests.NewNameTopic)
+	defer bus.Stop()
+
+	pool := resolvers.NewResolverPool(nil)
+	defer pool.Stop()
+
+	srv := NewPassiveTotal(cfg, bus, pool)
+
+	result := testService(srv, out)
+	if result < expectedTest {
+		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
+	}
+}

From 9bb7ec615e058eca89946e1d8816f3d030ffe7a8 Mon Sep 17 00:00:00 2001
From: vltraheaven <vltraheaven@protonmail.com>
Date: Tue, 30 Jul 2019 23:30:41 -0700
Subject: [PATCH 27/33] Created alienvault_test.go unit test file for
 AlienVault source

---
 services/sources/alienvault_test.go | 38 +++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 services/sources/alienvault_test.go

diff --git a/services/sources/alienvault_test.go b/services/sources/alienvault_test.go
new file mode 100644
index 000000000..b41ae8bff
--- /dev/null
+++ b/services/sources/alienvault_test.go
@@ -0,0 +1,38 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/OWASP/Amass/config"
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+)
+
+func TestAlienVault(t *testing.T) {
+	if *networkTest == false || *configPath == "" {
+		return
+	}
+
+	cfg := setupConfig(domainTest)
+
+	API := new(config.APIKey)
+	API = cfg.GetAPIKey("alienvault")
+
+	if API == nil || API.Key == "" {
+		t.Errorf("API key data was not provided")
+		return
+	}
+
+	bus, out := setupEventBus(requests.NewNameTopic)
+	defer bus.Stop()
+
+	pool := resolvers.NewResolverPool(nil)
+	defer pool.Stop()
+
+	srv := NewAlienVault(cfg, bus, pool)
+
+	result := testService(srv, out)
+	if result < expectedTest {
+		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
+	}
+}

From 97b636c3b756a9cb9e3b1396db75113b6216574a Mon Sep 17 00:00:00 2001
From: vltraheaven <vltraheaven@protonmail.com>
Date: Tue, 30 Jul 2019 23:35:54 -0700
Subject: [PATCH 28/33] Created binaryedge_test.go unit test file for
 BinaryEdge source

---
 services/sources/binaryedge_test.go | 38 +++++++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 services/sources/binaryedge_test.go

diff --git a/services/sources/binaryedge_test.go b/services/sources/binaryedge_test.go
new file mode 100644
index 000000000..8955011de
--- /dev/null
+++ b/services/sources/binaryedge_test.go
@@ -0,0 +1,38 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/OWASP/Amass/config"
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+)
+
+func TestBinaryEdge(t *testing.T) {
+	if *networkTest == false || *configPath == "" {
+		return
+	}
+
+	cfg := setupConfig(domainTest)
+
+	API := new(config.APIKey)
+	API = cfg.GetAPIKey("binaryedge")
+
+	if API == nil || API.Key == "" {
+		t.Errorf("API key data was not provided")
+		return
+	}
+
+	bus, out := setupEventBus(requests.NewNameTopic)
+	defer bus.Stop()
+
+	pool := resolvers.NewResolverPool(nil)
+	defer pool.Stop()
+
+	srv := NewBinaryEdge(cfg, bus, pool)
+
+	result := testService(srv, out)
+	if result < expectedTest {
+		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
+	}
+}

From 8be762663bb99cc53eeee4a89b9b91add370aa9c Mon Sep 17 00:00:00 2001
From: vltraheaven <vltraheaven@protonmail.com>
Date: Tue, 30 Jul 2019 23:39:06 -0700
Subject: [PATCH 29/33] Created securitytrails_test.go unit test file for 
 source

---
 services/sources/securitytrails_test.go | 38 +++++++++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 services/sources/securitytrails_test.go

diff --git a/services/sources/securitytrails_test.go b/services/sources/securitytrails_test.go
new file mode 100644
index 000000000..f4941c5b5
--- /dev/null
+++ b/services/sources/securitytrails_test.go
@@ -0,0 +1,38 @@
+package sources
+
+import (
+	"testing"
+
+	"github.com/OWASP/Amass/config"
+	"github.com/OWASP/Amass/requests"
+	"github.com/OWASP/Amass/resolvers"
+)
+
+func TestSecurityTrails(t *testing.T) {
+	if *networkTest == false || *configPath == "" {
+		return
+	}
+
+	cfg := setupConfig(domainTest)
+
+	API := new(config.APIKey)
+	API = cfg.GetAPIKey("securitytrails")
+
+	if API == nil || API.Key == "" {
+		t.Errorf("API key data was not provided")
+		return
+	}
+	
+	bus, out := setupEventBus(requests.NewNameTopic)
+	defer bus.Stop()
+
+	pool := resolvers.NewResolverPool(nil)
+	defer pool.Stop()
+
+	srv := NewSecurityTrails(cfg, bus, pool)
+
+	result := testService(srv, out)
+	if result < expectedTest {
+		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
+	}
+}
\ No newline at end of file

From d45e8da8d7102e831a4b515e2b5c3863a9f41b00 Mon Sep 17 00:00:00 2001
From: fork-while-fork <git@norse.pw>
Date: Wed, 31 Jul 2019 15:33:11 -0400
Subject: [PATCH 30/33] Fix viewdns URLs to use HTTPS. Fixes #235

---
 services/sources/viewdns.go | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/services/sources/viewdns.go b/services/sources/viewdns.go
index 66a67b3b5..5b5f3eb79 100644
--- a/services/sources/viewdns.go
+++ b/services/sources/viewdns.go
@@ -80,7 +80,7 @@ func (v *ViewDNS) processRequests() {
 func (v *ViewDNS) executeDNSQuery(domain string) {
 	var unique []string
 
-	u := "http://viewdns.info/iphistory/?domain=" + domain
+	u := v.getIPHistoryURL(domain)
 	// The ViewDNS IP History lookup sometimes reveals interesting results
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
@@ -105,7 +105,7 @@ func (v *ViewDNS) executeDNSQuery(domain string) {
 }
 
 func (v *ViewDNS) executeWhoisQuery(domain string) {
-	u := v.getURL(domain)
+	u := v.getReverseWhoisURL(domain)
 	page, err := utils.RequestWebPage(u, nil, nil, "", "")
 	if err != nil {
 		v.Bus().Publish(requests.LogTopic, fmt.Sprintf("%s: %s: %v", v.String(), u, err))
@@ -141,12 +141,6 @@ func (v *ViewDNS) executeWhoisQuery(domain string) {
 	}
 }
 
-func (v *ViewDNS) getURL(domain string) string {
-	format := "http://viewdns.info/reversewhois/?q=%s"
-
-	return fmt.Sprintf(format, domain)
-}
-
 func getViewDNSTable(page string) string {
 	var begin, end int
 
@@ -169,3 +163,13 @@ func getViewDNSTable(page string) string {
 	i = strings.Index(page[begin+i+6:end], "<table")
 	return page[begin+i : end]
 }
+
+func (v *ViewDNS) getReverseWhoisURL(domain string) string {
+	format := "https://viewdns.info/reversewhois/?q=%s"
+	return fmt.Sprintf(format, domain)
+}
+
+func (v *ViewDNS) getIPHistoryURL(domain string) string {
+	format := "https://viewdns.info/iphistory/?domain=%s"
+	return fmt.Sprintf(format, domain)
+}

From 2b32ac514236f22595e4e89fb7c14a16eed4df29 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 1 Aug 2019 15:20:39 -0400
Subject: [PATCH 31/33] added comments and made format changes

---
 services/sources/commoncrawl.go    |  4 ++--
 services/sources/hackerone_test.go |  4 ++--
 services/sources/shodan_test.go    |  4 ++--
 stringset/set.go                   | 20 +++++++++++++++++++-
 4 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/services/sources/commoncrawl.go b/services/sources/commoncrawl.go
index 23f9d06d5..358677b1d 100644
--- a/services/sources/commoncrawl.go
+++ b/services/sources/commoncrawl.go
@@ -44,7 +44,7 @@ func (c *CommonCrawl) OnStart() error {
 	// Get all of the index API URLs
 	page, err := utils.RequestWebPage(commonCrawlIndexListURL, nil, nil, "", "")
 	if err != nil {
-		c.Bus().Publish(requests.LogTopic, 
+		c.Bus().Publish(requests.LogTopic,
 			fmt.Sprintf("%s: Failed to obtain the index list: %v", c.String(), err),
 		)
 		return fmt.Errorf("%s: Failed to obtain the index list: %v", c.String(), err)
@@ -58,7 +58,7 @@ func (c *CommonCrawl) OnStart() error {
 
 	var indexList []index
 	if err := json.Unmarshal([]byte(page), &indexList); err != nil {
-		c.Bus().Publish(requests.LogTopic, 
+		c.Bus().Publish(requests.LogTopic,
 			fmt.Sprintf("%s: Failed to unmarshal the index list: %v", c.String(), err),
 		)
 		return fmt.Errorf("%s: Failed to unmarshal the index list: %v", c.String(), err)
diff --git a/services/sources/hackerone_test.go b/services/sources/hackerone_test.go
index a015c7223..e1da64ea3 100644
--- a/services/sources/hackerone_test.go
+++ b/services/sources/hackerone_test.go
@@ -15,7 +15,7 @@ func TestHackerone(t *testing.T) {
 	domainTest = "twitter.com"
 
 	cfg := setupConfig(domainTest)
-	
+
 	bus, out := setupEventBus(requests.NewNameTopic)
 	defer bus.Stop()
 
@@ -28,4 +28,4 @@ func TestHackerone(t *testing.T) {
 	if result < expectedTest {
 		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
 	}
-}
\ No newline at end of file
+}
diff --git a/services/sources/shodan_test.go b/services/sources/shodan_test.go
index f20a2c84a..7b4c7bcfb 100644
--- a/services/sources/shodan_test.go
+++ b/services/sources/shodan_test.go
@@ -22,7 +22,7 @@ func TestShodan(t *testing.T) {
 		t.Errorf("API key data was not provided")
 		return
 	}
-	
+
 	bus, out := setupEventBus(requests.NewNameTopic)
 	defer bus.Stop()
 
@@ -35,4 +35,4 @@ func TestShodan(t *testing.T) {
 	if result < expectedTest {
 		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
 	}
-}
\ No newline at end of file
+}
diff --git a/stringset/set.go b/stringset/set.go
index e038ef72c..227938782 100644
--- a/stringset/set.go
+++ b/stringset/set.go
@@ -6,15 +6,18 @@ import (
 )
 
 type (
+	// Set implements set operations for string values.
 	Set map[string]nothing
 
 	nothing struct{}
 )
 
+// Deduplicate utilizes the Set type to generate a unique list of strings from the input slice.
 func Deduplicate(input []string) []string {
 	return New(input...).Slice()
 }
 
+// New returns a Set containing the values provided in the arguments.
 func New(initial ...string) Set {
 	s := Set{}
 
@@ -25,25 +28,30 @@ func New(initial ...string) Set {
 	return s
 }
 
+// Has returns true if the receiver Set already contains the element string argument.
 func (s Set) Has(element string) bool {
 	_, exists := s[element]
 	return exists
 }
 
+// Insert adds the element string argument to the receiver Set.
 func (s Set) Insert(element string) {
 	s[strings.ToLower(element)] = nothing{}
 }
 
+// InsertMany adds all the elements strings into the receiver Set.
 func (s Set) InsertMany(elements ...string) {
 	for _, i := range elements {
 		s.Insert(i)
 	}
 }
 
+// Remove will delete the element string from the receiver Set.
 func (s Set) Remove(element string) {
 	delete(s, element)
 }
 
+// Slice returns a string slice that contains all the elements in the Set.
 func (s Set) Slice() []string {
 	var i uint64
 
@@ -57,16 +65,19 @@ func (s Set) Slice() []string {
 	return k
 }
 
+// Union adds all the elements from the other Set argument into the receiver Set.
 func (s Set) Union(other Set) {
 	for k := range other {
 		s.Insert(k)
 	}
 }
 
+// Len returns the number of elements in the receiver Set.
 func (s Set) Len() int {
 	return len(s)
 }
 
+// Subtract removes all elements in the other Set argument from the receiver Set.
 func (s Set) Subtract(other Set) {
 	for item := range other {
 		if s.Has(item) {
@@ -75,6 +86,8 @@ func (s Set) Subtract(other Set) {
 	}
 }
 
+// Intersect causes the receiver Set to only contains elements also found in the
+// other Set argument.
 func (s Set) Intersect(other Set) {
 	intersect := New()
 
@@ -84,11 +97,16 @@ func (s Set) Intersect(other Set) {
 		}
 	}
 
+	var remove []string
 	for item := range s {
 		if !intersect.Has(item) {
-			s.Remove(item)
+			remove = append(remove, item)
 		}
 	}
+
+	for _, r := range remove {
+		s.Remove(r)
+	}
 }
 
 // Set implements the flag.Value interface.

From a59e2c8eadd6f6556a65366bb67b522cc0489e35 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 1 Aug 2019 18:08:46 -0400
Subject: [PATCH 32/33] fixed incorrect variable names

---
 services/sources/alienvault_test.go     |  8 ++------
 services/sources/binaryedge_test.go     |  8 ++------
 services/sources/passivetotal_test.go   |  8 ++------
 services/sources/securitytrails_test.go | 12 ++++--------
 services/sources/shodan_test.go         |  8 ++------
 services/sources/spyse_test.go          |  8 ++------
 services/sources/twitter_test.go        |  8 ++------
 7 files changed, 16 insertions(+), 44 deletions(-)

diff --git a/services/sources/alienvault_test.go b/services/sources/alienvault_test.go
index b41ae8bff..8c96e35c1 100644
--- a/services/sources/alienvault_test.go
+++ b/services/sources/alienvault_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,11 +13,8 @@ func TestAlienVault(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("alienvault")
-
-	if API == nil || API.Key == "" {
+	api := cfg.GetAPIKey("alienvault")
+	if api == nil || api.Key == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}
diff --git a/services/sources/binaryedge_test.go b/services/sources/binaryedge_test.go
index 8955011de..a5fc81352 100644
--- a/services/sources/binaryedge_test.go
+++ b/services/sources/binaryedge_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,11 +13,8 @@ func TestBinaryEdge(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("binaryedge")
-
-	if API == nil || API.Key == "" {
+	api := cfg.GetAPIKey("binaryedge")
+	if api == nil || api.Key == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}
diff --git a/services/sources/passivetotal_test.go b/services/sources/passivetotal_test.go
index a1dff5bf2..9ada6f78f 100644
--- a/services/sources/passivetotal_test.go
+++ b/services/sources/passivetotal_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,11 +13,8 @@ func TestPassiveTotal(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("passivetotal")
-
-	if API == nil || API.Username == "" || API.Key == "" {
+	api := cfg.GetAPIKey("passivetotal")
+	if api == nil || api.Username == "" || api.Key == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}
diff --git a/services/sources/securitytrails_test.go b/services/sources/securitytrails_test.go
index f4941c5b5..d5d71798d 100644
--- a/services/sources/securitytrails_test.go
+++ b/services/sources/securitytrails_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,15 +13,12 @@ func TestSecurityTrails(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("securitytrails")
-
-	if API == nil || API.Key == "" {
+	api := cfg.GetAPIKey("securitytrails")
+	if api == nil || api.Key == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}
-	
+
 	bus, out := setupEventBus(requests.NewNameTopic)
 	defer bus.Stop()
 
@@ -35,4 +31,4 @@ func TestSecurityTrails(t *testing.T) {
 	if result < expectedTest {
 		t.Errorf("Found %d names, expected at least %d instead", result, expectedTest)
 	}
-}
\ No newline at end of file
+}
diff --git a/services/sources/shodan_test.go b/services/sources/shodan_test.go
index 7b4c7bcfb..bdea99946 100644
--- a/services/sources/shodan_test.go
+++ b/services/sources/shodan_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,11 +13,8 @@ func TestShodan(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("shodan")
-
-	if API == nil || API.Key == "" {
+	api := cfg.GetAPIKey("shodan")
+	if api == nil || api.Key == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}
diff --git a/services/sources/spyse_test.go b/services/sources/spyse_test.go
index 9786d2f98..7361b933d 100644
--- a/services/sources/spyse_test.go
+++ b/services/sources/spyse_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,11 +13,8 @@ func TestSpyse(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("spyse")
-
-	if API == nil || API.Key == "" || API.Secret == "" {
+	api := cfg.GetAPIKey("spyse")
+	if api == nil || api.Key == "" || api.Secret == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}
diff --git a/services/sources/twitter_test.go b/services/sources/twitter_test.go
index 3157097df..a8621e92d 100644
--- a/services/sources/twitter_test.go
+++ b/services/sources/twitter_test.go
@@ -3,7 +3,6 @@ package sources
 import (
 	"testing"
 
-	"github.com/OWASP/Amass/config"
 	"github.com/OWASP/Amass/requests"
 	"github.com/OWASP/Amass/resolvers"
 )
@@ -14,11 +13,8 @@ func TestTwitter(t *testing.T) {
 	}
 
 	cfg := setupConfig(domainTest)
-
-	API := new(config.APIKey)
-	API = cfg.GetAPIKey("twitter")
-
-	if API == nil || API.Key == "" || API.Secret == "" {
+	api := cfg.GetAPIKey("twitter")
+	if api == nil || api.Key == "" || api.Secret == "" {
 		t.Errorf("API key data was not provided")
 		return
 	}

From 77ed1c04844d3a89c593d58a45a4be94b4a41c81 Mon Sep 17 00:00:00 2001
From: caffix <caffix@gmail.com>
Date: Thu, 1 Aug 2019 18:12:29 -0400
Subject: [PATCH 33/33] added the DarkReading mention

---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index e4d4706b8..9322f7ca1 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,7 @@ This project improves thanks to all the people who contribute:
 
 ## Mentions
 
+* [8 Free Tools to Be Showcased at Black Hat and DEF CON](https://www.darkreading.com/application-security/8-free-tools-to-be-showcased-at-black-hat-and-def-con/d/d-id/1335356?image_number=5)
 * [amass — Automated Attack Surface Mapping](https://danielmiessler.com/study/amass/)
 * [Aquatone — A Tool for Domain Flyovers](https://github.com/michenriksen/aquatone)
 * [Collaborating with the Crowd – Recapping LevelUp 0X04](https://www.bugcrowd.com/blog/recapping_levelup_0x04/)