architecture handles all network info in the Address Service now

amass/activecert.go

@@ -158,7 +158,7 @@ func reqFromNames(subdomains []string) []*core.DNSRequest {
 	for _, name := range subdomains {
 		requests = append(requests, &core.DNSRequest{
 			Name:   name,
-			Domain: SubdomainToDomain(name),
+			Domain: core.SubdomainToDomain(name),
 			Tag:    core.CERT,
 		})
 	}

amass/addrsrv.go

@@ -4,6 +4,11 @@
 package amass
 
 import (
+	"net"
+	"sync"
+	"errors"
+	"time"
+
 	"github.com/OWASP/Amass/amass/core"
 	"github.com/OWASP/Amass/amass/utils"
 )
@@ -14,11 +19,25 @@ type AddressService struct {
 	core.BaseService
 
 	filter *utils.StringFilter
+	// The private network address ranges
+	private192 *net.IPNet
+	private172 *net.IPNet
+	private10  *net.IPNet
+	// Cache for the infrastructure data collected from online sources
+	netLock  sync.Mutex
+	netCache map[int]*core.ASNRequest
 }
 
 // NewAddressService returns he object initialized, but not yet started.
 func NewAddressService(config *core.Config, bus *core.EventBus) *AddressService {
-	as := &AddressService{filter: utils.NewStringFilter()}
+	as := &AddressService{
+		filter: utils.NewStringFilter(),
+		netCache: make(map[int]*core.ASNRequest),
+	}
+
+	_, as.private192, _ = net.ParseCIDR("192.168.0.0/16")
+	_, as.private172, _ = net.ParseCIDR("172.16.0.0/12")
+	_, as.private10, _ = net.ParseCIDR("10.0.0.0/8")
 
 	as.BaseService = *core.NewBaseService(as, "Address Service", config, bus)
 	return as
@@ -29,6 +48,8 @@ func (as *AddressService) OnStart() error {
 	as.BaseService.OnStart()
 
 	as.Bus().Subscribe(core.NewAddrTopic, as.SendAddrRequest)
+	as.Bus().Subscribe(core.NewASNTopic, as.SendASNRequest)
+	as.Bus().Subscribe(core.IPRequestTopic, as.performIPRequest)
 	go as.processRequests()
 	return nil
 }
@@ -40,16 +61,17 @@ func (as *AddressService) processRequests() {
 			<-as.ResumeChan()
 		case <-as.Quit():
 			return
-		case req := <-as.AddrRequestChan():
-			go as.performRequest(req)
+		case addr := <-as.AddrRequestChan():
+			go as.performAddrRequest(addr)
+		case asn := <-as.ASNRequestChan():
+			go as.performASNRequest(asn)
 		case <-as.DNSRequestChan():
-		case <-as.ASNRequestChan():
 		case <-as.WhoisRequestChan():
 		}
 	}
 }
 
-func (as *AddressService) performRequest(req *core.AddrRequest) {
+func (as *AddressService) performAddrRequest(req *core.AddrRequest) {
 	if req == nil || req.Address == "" {
 		return
 	}
@@ -60,10 +82,173 @@ func (as *AddressService) performRequest(req *core.AddrRequest) {
 	}
 	as.Bus().Publish(core.ActiveCertTopic, req)
 
-	_, cidr, _, err := IPRequest(req.Address)
-	if err != nil {
-		as.Config().Log.Printf("%v", err)
+	asn := as.ipSearch(req.Address)
+	if asn == nil {
+		return
+	}
+	if _, cidr, _ := net.ParseCIDR(asn.Prefix); cidr != nil {
+		as.Bus().Publish(core.ReverseSweepTopic, req.Address, cidr)
+	}
+}
+
+func (as *AddressService) performASNRequest(req *core.ASNRequest) {
+	as.netLock.Lock()
+	defer as.netLock.Unlock()
+
+	as.SetActive()
+	if _, found := as.netCache[req.ASN]; !found {
+		as.netCache[req.ASN] = req
+		return
+	}
+
+	c := as.netCache[req.ASN]
+	// This is additional information for an ASN entry
+	if c.Prefix == "" && req.Prefix != "" {
+		c.Prefix = req.Prefix
+	}
+	if c.CC == "" && req.CC != "" {
+		c.CC = req.CC
+	}
+	if c.Registry == "" && req.Registry != "" {
+		c.Registry = req.Registry
+	}
+	if c.AllocationDate.IsZero() && !req.AllocationDate.IsZero() {
+		c.AllocationDate = req.AllocationDate
+	}
+	if c.Description == "" && req.Description != "" {
+		c.Description = req.Description
+	}
+	c.Netblocks = utils.UniqueAppend(c.Netblocks, req.Netblocks...)
+}
+
+func (as *AddressService) performIPRequest(req *core.ASNRequest) {
+	if req.Address == "" {
+		return
+	}
+
+	as.SetActive()
+	// Does the address fall into a private network range?
+	if asn := as.checkForPrivateAddress(req.Address); asn != nil {
+		as.Bus().Publish(core.IPInfoTopic, asn)
 		return
 	}
-	as.Bus().Publish(core.ReverseSweepTopic, req.Address, cidr)
+	// Is the data already available in the cache?
+	if asn := as.ipSearch(req.Address); asn != nil {
+		as.Bus().Publish(core.IPInfoTopic, asn)
+		return
+	}
+	// Ask the data sources for the ASN information
+	as.Bus().Publish(core.IPToASNTopic, req)
+	// Wait for the results to hit the cache
+	for i := 0; i < 10; i++ {
+		time.Sleep(time.Second)
+		if asn := as.ipSearch(req.Address); asn != nil {
+			as.Bus().Publish(core.IPInfoTopic, asn)
+			return
+		}
+	}
+}
+
+func (as *AddressService) ipSearch(addr string) *core.ASNRequest {
+	as.netLock.Lock()
+	defer as.netLock.Unlock()
+
+	var a int
+	var cidr *net.IPNet
+	var desc string
+	ip := net.ParseIP(addr)
+	for asn, record := range as.netCache {
+		for _, netblock := range record.Netblocks {
+			_, ipnet, err := net.ParseCIDR(netblock)
+			if err != nil {
+				continue
+			}
+
+			if ipnet.Contains(ip) {
+				// Select the smallest CIDR
+				if cidr != nil && compareCIDRSizes(cidr, ipnet) == 1 {
+					continue
+				}
+				a = asn
+				cidr = ipnet
+				desc = record.Description
+			}
+		}
+	}
+	if cidr == nil {
+		return nil
+	}
+	return &core.ASNRequest{
+		Address: addr,
+		ASN: a,
+		Prefix: cidr.String(),
+		Description: desc,
+	}
+}
+
+func (as *AddressService) checkForPrivateAddress(addr string) *core.ASNRequest {
+	var n string
+	ip := net.ParseIP(addr)
+	desc := "Private Networks"
+
+	if as.private192.Contains(ip) {
+		n = as.private192.String()
+	} else if as.private172.Contains(ip) {
+		n = as.private172.String()
+	} else if as.private10.Contains(ip) {
+		n = as.private10.String()
+	}
+
+	if n == "" {
+		return nil
+	}
+	return &core.ASNRequest{
+		Address: addr,
+		Prefix: n,
+		Description: desc,
+	}
+}
+
+func compareCIDRSizes(first, second *net.IPNet) int {
+	var result int
+
+	s1, _ := first.Mask.Size()
+	s2, _ := second.Mask.Size()
+	if s1 > s2 {
+		result = 1
+	} else if s2 > s1 {
+		result = -1
+	}
+	return result
+}
+
+// IPRequest returns the ASN, CIDR and AS Description that contains the provided IP address.
+func IPRequest(addr string, bus *core.EventBus) (int, *net.IPNet, string, error) {
+	asnchan := make(chan *core.ASNRequest, 1)
+	f := func(req *core.ASNRequest) {
+		if req.Address == addr {
+			asnchan <- req
+		}
+	}
+
+	bus.Subscribe(core.IPInfoTopic, f)
+	defer bus.Unsubscribe(core.IPInfoTopic, f)
+	bus.Publish(core.IPRequestTopic, &core.ASNRequest{Address: addr})
+
+	var a int
+	var cidr *net.IPNet
+	var desc string
+	t := time.NewTimer(5 * time.Second)
+	select {
+	case <-t.C:
+	case asn := <-asnchan:
+		a = asn.ASN
+		_, cidr, _ = net.ParseCIDR(asn.Prefix)
+		desc = asn.Description
+	}
+
+	if cidr == nil {
+		return 0, nil, "", errors.New("Failed to obtain the IP information")
+	}
+	return a, cidr, desc, nil
 }

amass/alteration_test.go

@@ -14,8 +14,8 @@ import (
 )
 
 var (
-	alterationTestRequests = []*core.Request{
-		&core.Request{
+	alterationTestRequests = []*core.DNSRequest{
+		&core.DNSRequest{
 			Name:    "test1.owasp.org",
 			Domain:  "owasp.org",
 			Records: []core.DNSAnswer{core.DNSAnswer{Type: int(dns.TypeA)}},
@@ -40,21 +40,21 @@ func setupConfig(domain string) *core.Config {
 	return config
 }
 
-func setupEventBus(subscription string) (*core.EventBus, chan *core.Request) {
-	out := make(chan *core.Request)
+func setupEventBus(subscription string) (*core.EventBus, chan *core.DNSRequest) {
+	out := make(chan *core.DNSRequest)
 	bus := core.NewEventBus()
-	bus.Subscribe(subscription, func(req *core.Request) {
+	bus.Subscribe(subscription, func(req *core.DNSRequest) {
 		out <- req
 	})
 
 	return bus, out
 }
 
-func testService(srv core.Service, out chan *core.Request) int {
+func testService(srv core.Service, out chan *core.DNSRequest) int {
 	srv.Start()
 	defer srv.Stop()
 
-	srv.SendRequest(alterationTestRequests[0])
+	srv.SendDNSRequest(alterationTestRequests[0])
 
 	count := 0
 	doneTimer := time.After(time.Second * 3)
@@ -92,13 +92,13 @@ func TestAlterations(t *testing.T) {
 
 func TestCorrectRecordTypes(t *testing.T) {
 	var (
-		alterationTestRequests = []*core.Request{
-			&core.Request{
+		alterationTestRequests = []*core.DNSRequest{
+			&core.DNSRequest{
 				Name:    "test1.owasp.org",
 				Domain:  "owasp.org",
 				Records: []core.DNSAnswer{core.DNSAnswer{Type: int(dns.TypeA)}},
 			},
-			&core.Request{
+			&core.DNSRequest{
 				Name:    "test.twitter.com",
 				Domain:  "twitter.com",
 				Records: []core.DNSAnswer{core.DNSAnswer{Type: int(dns.TypeA)}},

amass/brute.go

@@ -179,7 +179,7 @@ func (bfs *BruteForceService) bruteForceResolution(word, sub, domain string) {
 	name := word + "." + sub
 	var answers []core.DNSAnswer
 	for _, t := range BruteForceQueryTypes {
-		if a, err := Resolve(name, t, PriorityLow); err == nil {
+		if a, err := core.Resolve(name, t, core.PriorityLow); err == nil {
 			answers = append(answers, a...)
 			// Do not continue if a CNAME was discovered
 			if t == "CNAME" {

amass/core/request.go

@@ -35,7 +35,10 @@ const (
 	ReverseSweepTopic = "amass:sweep"
 	ActiveCertTopic   = "amass:activecert"
 	OutputTopic       = "amass:output"
+	IPToASNTopic       = "amass:iptoasn"
 	NewASNTopic       = "amass:asn"
+	IPRequestTopic = "amass:iprequest"
+	IPInfoTopic = "amass:ipinfo"
 	NewWhoisTopic     = "amass:whois"
 )
 
@@ -66,20 +69,25 @@ type AddrRequest struct {
 
 // ASNRequest handles all autonomous system information needed by Amass.
 type ASNRequest struct {
+	Address string
 	ASN            int
 	Prefix         string
 	CC             string
 	Registry       string
 	AllocationDate time.Time
 	Description    string
 	Netblocks      []string
+	Tag     string
+	Source  string
 }
 
 // WhoisRequest handles data needed throughout Service processing of reverse whois.
 type WhoisRequest struct {
-	Name    string
+	Domain    string
 	Company string
 	Email   string
+	Tag     string
+	Source  string
 }
 
 // Output contains all the output data for an enumerated DNS name.
@@ -127,7 +135,7 @@ func NewEventBus() *EventBus {
 	return eb
 }
 
-// Subscribe registers callback to be executed for all requests on the channel labeled name.
+// Subscribe registers callback to be executed for all requests on the channel.
 func (eb *EventBus) Subscribe(topic string, fn interface{}) {
 	if topic == "" || reflect.TypeOf(fn).Kind() != reflect.Func {
 		return
@@ -140,6 +148,26 @@ func (eb *EventBus) Subscribe(topic string, fn interface{}) {
 	eb.Unlock()
 }
 
+// Unsubscribe deregisters the callback from the channel.
+func (eb *EventBus) Unsubscribe(topic string, fn interface{}) {
+	if topic == "" || reflect.TypeOf(fn).Kind() != reflect.Func {
+		return
+	}
+
+	callback := reflect.ValueOf(fn)
+
+	eb.Lock()
+	defer eb.Unlock()
+
+	var channels []reflect.Value
+	for _, c := range eb.topics[topic] {
+		if c != callback {
+			channels = append(channels, c)
+		}
+	}
+	eb.topics[topic] = channels
+}
+
 // Publish sends req on the channel labeled with name.
 func (eb *EventBus) Publish(topic string, args ...interface{}) {
 	if topic == "" {