Skip to content

Commit

Permalink
Merge branch 'develop' of https://github.com/OWASP/Amass
Browse files Browse the repository at this point in the history
  • Loading branch information
@caffix
caffix committed Aug 1, 2019
2 parents acd30fd + 77ed1c0 commit d095c1f
Show file tree
Hide file tree
Showing 90 changed files with 1,468 additions and 762 deletions.
7 changes: 5 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ The OWASP Amass Project has developed a tool to help information security profes
**Information Gathering Techniques Used:**

* **DNS:** Basic enumeration, Brute forcing (upon request), Reverse DNS sweeping, Subdomain name alterations/permutations, Zone transfers (upon request)
* **Scraping:** Ask, Baidu, Bing, CommonCrawl, DNSDumpster, DNSTable, Dogpile, Exalead, Google, HackerOne, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ViewDNS, Yahoo
* **Scraping:** Ask, Baidu, Bing, DNSDumpster, DNSTable, Dogpile, Exalead, Google, HackerOne, IPv4Info, Netcraft, PTRArchive, Riddler, SiteDossier, ViewDNS, Yahoo
* **Certificates:** Active pulls (upon request), Censys, CertSpotter, Crtsh, Entrust, GoogleCT
* **APIs:** AlienVault, BinaryEdge, BufferOver, CIRCL, DNSDB, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse (CertDB & FindSubdomains), Sublist3rAPI, TeamCymru, ThreatCrowd, Twitter, Umbrella, URLScan, VirusTotal
* **APIs:** AlienVault, BinaryEdge, BufferOver, CIRCL, CommonCrawl, DNSDB, HackerTarget, Mnemonic, NetworksDB, PassiveTotal, RADb, Robtex, SecurityTrails, ShadowServer, Shodan, Spyse (CertDB & FindSubdomains), Sublist3rAPI, TeamCymru, ThreatCrowd, Twitter, Umbrella, URLScan, VirusTotal
* **Web Archives:** ArchiveIt, ArchiveToday, Arquivo, LoCArchive, OpenUKArchive, UKGovArchive, Wayback

----
Expand Down Expand Up @@ -68,10 +68,13 @@ This project improves thanks to all the people who contribute:
[![Follow on Twitter](https://img.shields.io/twitter/follow/sec_for_safety.svg?logo=twitter)](https://twitter.com/sec_for_safety)
[![Follow on Twitter](https://img.shields.io/twitter/follow/ngkogkos.svg?logo=twitter)](https://github.com/ngkogkos)
[![Follow on Twitter](https://img.shields.io/twitter/follow/Jhaddix.svg?logo=twitter)](https://twitter.com/Jhaddix)
[![Follow on Twitter](https://img.shields.io/twitter/follow/Vltraheaven.svg?logo=twitter)](https://twitter.com/Vltraheaven)

## Mentions

* [8 Free Tools to Be Showcased at Black Hat and DEF CON](https://www.darkreading.com/application-security/8-free-tools-to-be-showcased-at-black-hat-and-def-con/d/d-id/1335356?image_number=5)
* [amass — Automated Attack Surface Mapping](https://danielmiessler.com/study/amass/)
* [Aquatone — A Tool for Domain Flyovers](https://github.com/michenriksen/aquatone)
* [Collaborating with the Crowd – Recapping LevelUp 0X04](https://www.bugcrowd.com/blog/recapping_levelup_0x04/)
* [Subdomain Enumeration: 2019 Workflow](https://0xpatrik.com/subdomain-enumeration-2019/)
* [REMOTE CODE EXECUTION ! 😜 Recon Wins](https://medium.com/@vishnu0002/remote-code-execution-recon-wins-e9c1db79f3da)
Expand Down
19 changes: 12 additions & 7 deletions cmd/amass/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/OWASP/Amass/config"
"github.com/OWASP/Amass/graph"
"github.com/OWASP/Amass/requests"
"github.com/OWASP/Amass/stringset"
"github.com/OWASP/Amass/utils"
"github.com/fatih/color"
)
Expand All @@ -25,7 +26,7 @@ const (
)

type dbArgs struct {
Domains utils.ParseStrings
Domains stringset.Set
Enum int
Options struct {
DemoMode bool
Expand All @@ -52,6 +53,8 @@ func runDBCommand(clArgs []string) {
dbBuf := new(bytes.Buffer)
dbCommand.SetOutput(dbBuf)

args.Domains = stringset.New()

dbCommand.BoolVar(&help1, "h", false, "Show the program usage message")
dbCommand.BoolVar(&help2, "help", false, "Show the program usage message")
dbCommand.Var(&args.Domains, "d", "Domain names separated by commas (can be used multiple times)")
Expand Down Expand Up @@ -88,7 +91,7 @@ func runDBCommand(clArgs []string) {
r.Fprintf(color.Error, "Failed to parse the domain names file: %v\n", err)
return
}
args.Domains = utils.UniqueAppend(args.Domains, list...)
args.Domains.InsertMany(list...)
}

cfg := new(config.Config)
Expand All @@ -98,7 +101,7 @@ func runDBCommand(clArgs []string) {
args.Filepaths.Directory = cfg.Dir
}
if len(args.Domains) == 0 {
args.Domains = utils.UniqueAppend(args.Domains, cfg.Domains()...)
args.Domains.InsertMany(cfg.Domains()...)
}
} else if args.Filepaths.ConfigFile != "" {
r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
Expand All @@ -122,7 +125,7 @@ func runDBCommand(clArgs []string) {
}

if args.Options.ListEnumerations {
listEnumerations(args.Domains, db)
listEnumerations(&args, db)
return
}

Expand Down Expand Up @@ -172,7 +175,8 @@ func inputDataOperations(args *dbArgs, db graph.DataHandler) error {
return nil
}

func listEnumerations(domains []string, db graph.DataHandler) {
func listEnumerations(args *dbArgs, db graph.DataHandler) {
domains := args.Domains.Slice()
enums := enumIDs(domains, db)
if len(enums) == 0 {
r.Fprintln(color.Error, "No enumerations found within the provided scope")
Expand All @@ -198,11 +202,12 @@ func listEnumerations(domains []string, db graph.DataHandler) {
}

func showEnumeration(args *dbArgs, db graph.DataHandler) {
domains := args.Domains.Slice()
var total int
tags := make(map[string]int)
asns := make(map[int]*utils.ASNSummaryData)
for _, out := range getEnumOutput(args.Enum, args.Domains, db) {
if len(args.Domains) > 0 && !domainNameInScope(out.Name, args.Domains) {
for _, out := range getEnumOutput(args.Enum, domains, db) {
if len(domains) > 0 && !domainNameInScope(out.Name, domains) {
continue
}

Expand Down
143 changes: 59 additions & 84 deletions cmd/amass/enum.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (

"github.com/OWASP/Amass/config"
"github.com/OWASP/Amass/enum"
"github.com/OWASP/Amass/stringset"
"github.com/OWASP/Amass/utils"
"github.com/fatih/color"
homedir "github.com/mitchellh/go-homedir"
Expand All @@ -40,19 +41,19 @@ type enumArgs struct {
Addresses utils.ParseIPs
ASNs utils.ParseInts
CIDRs utils.ParseCIDRs
AltWordList []string
AltWordListMask utils.ParseStrings
BruteWordList []string
BruteWordListMask utils.ParseStrings
Blacklist utils.ParseStrings
Domains utils.ParseStrings
Excluded utils.ParseStrings
Included utils.ParseStrings
AltWordList stringset.Set
AltWordListMask stringset.Set
BruteWordList stringset.Set
BruteWordListMask stringset.Set
Blacklist stringset.Set
Domains stringset.Set
Excluded stringset.Set
Included stringset.Set
MaxDNSQueries int
MinForRecursive int
Names []string
Names stringset.Set
Ports utils.ParseInts
Resolvers utils.ParseStrings
Resolvers stringset.Set
Options struct {
Active bool
BruteForcing bool
Expand Down Expand Up @@ -136,7 +137,18 @@ func defineEnumFilepathFlags(enumFlags *flag.FlagSet, args *enumArgs) {
}

func runEnumCommand(clArgs []string) {
var args enumArgs
args := enumArgs{
AltWordList: stringset.New(),
AltWordListMask: stringset.New(),
BruteWordList: stringset.New(),
BruteWordListMask: stringset.New(),
Blacklist: stringset.New(),
Domains: stringset.New(),
Excluded: stringset.New(),
Included: stringset.New(),
Names: stringset.New(),
Resolvers: stringset.New(),
}
var help1, help2 bool
enumCommand := flag.NewFlagSet("enum", flag.ContinueOnError)

Expand Down Expand Up @@ -171,10 +183,10 @@ func runEnumCommand(clArgs []string) {
}

if len(args.AltWordListMask) > 0 {
args.AltWordList = utils.UniqueAppend(args.AltWordList, args.AltWordListMask...)
args.AltWordList.Union(args.AltWordListMask)
}
if len(args.BruteWordListMask) > 0 {
args.BruteWordList = utils.UniqueAppend(args.BruteWordList, args.BruteWordListMask...)
args.BruteWordList.Union(args.BruteWordListMask)
}
// Some input validation
if args.Options.Passive && (args.Options.IPs || args.Options.IPv4 || args.Options.IPv6) {
Expand Down Expand Up @@ -202,12 +214,12 @@ func runEnumCommand(clArgs []string) {

rLog, wLog := io.Pipe()
e.Config.Log = log.New(wLog, "", log.Lmicroseconds)

// Check if a configuration file was provided, and if so, load the settings
if f, err := config.AcquireConfig(args.Filepaths.Directory, args.Filepaths.ConfigFile, e.Config); err == nil {
// Check if a config file was provided that has DNS resolvers specified
if r, err := config.GetResolversFromSettings(f); err == nil && len(args.Resolvers) == 0 {
args.Resolvers = r
args.Resolvers = stringset.New(r...)
}
} else if args.Filepaths.ConfigFile != "" {
r.Fprintf(color.Error, "Failed to load the configuration file: %v\n", err)
Expand All @@ -221,7 +233,7 @@ func runEnumCommand(clArgs []string) {
}

if len(args.Resolvers) > 0 {
if err := e.Pool.SetResolvers(args.Resolvers); err != nil {
if err := e.Pool.SetResolvers(args.Resolvers.Slice()); err != nil {
r.Fprintf(color.Error, "Failed to set custom DNS resolvers: %v\n", err)
os.Exit(1)
}
Expand Down Expand Up @@ -435,7 +447,7 @@ func processEnumInputFiles(args *enumArgs) error {
return fmt.Errorf("Failed to parse the brute force wordlist file: %v", err)
}

args.BruteWordList = utils.UniqueAppend(args.BruteWordList, list...)
args.BruteWordList.InsertMany(list...)
}
}
if !args.Options.NoAlts && len(args.Filepaths.AltWordlist) > 0 {
Expand All @@ -445,29 +457,29 @@ func processEnumInputFiles(args *enumArgs) error {
return fmt.Errorf("Failed to parse the alterations wordlist file: %v", err)
}

args.AltWordList = utils.UniqueAppend(args.AltWordList, list...)
args.AltWordList.InsertMany(list...)
}
}
if args.Filepaths.Blacklist != "" {
list, err := config.GetListFromFile(args.Filepaths.Blacklist)
if err != nil {
return fmt.Errorf("Failed to parse the blacklist file: %v", err)
}
args.Blacklist = utils.UniqueAppend(args.Blacklist, list...)
args.Blacklist.InsertMany(list...)
}
if args.Filepaths.ExcludedSrcs != "" {
list, err := config.GetListFromFile(args.Filepaths.ExcludedSrcs)
if err != nil {
return fmt.Errorf("Failed to parse the exclude file: %v", err)
}
args.Excluded = utils.UniqueAppend(args.Excluded, list...)
args.Excluded.InsertMany(list...)
}
if args.Filepaths.IncludedSrcs != "" {
list, err := config.GetListFromFile(args.Filepaths.IncludedSrcs)
if err != nil {
return fmt.Errorf("Failed to parse the include file: %v", err)
}
args.Included = utils.UniqueAppend(args.Included, list...)
args.Included.InsertMany(list...)
}
if len(args.Filepaths.Names) > 0 {
for _, f := range args.Filepaths.Names {
Expand All @@ -476,7 +488,7 @@ func processEnumInputFiles(args *enumArgs) error {
return fmt.Errorf("Failed to parse the subdomain names file: %v", err)
}

args.Names = utils.UniqueAppend(args.Names, list...)
args.Names.InsertMany(list...)
}
}
if len(args.Filepaths.Domains) > 0 {
Expand All @@ -486,17 +498,17 @@ func processEnumInputFiles(args *enumArgs) error {
return fmt.Errorf("Failed to parse the domain names file: %v", err)
}

args.Domains = utils.UniqueAppend(args.Domains, list...)
args.Domains.InsertMany(list...)
}
}
if len(args.Filepaths.Resolvers) > 0 {
for _, f := range args.Filepaths.Resolvers {
list, err := config.GetListFromFile(f)
if err != nil {
return fmt.Errorf("Failed to parse the resolver file: %v", err)
return fmt.Errorf("Failed to parse the esolver file: %v", err)
}

args.Resolvers = utils.UniqueAppend(args.Resolvers, list...)
args.Resolvers.InsertMany(list...)
}
}
return nil
Expand All @@ -523,13 +535,13 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
e.Config.MaxDNSQueries = args.MaxDNSQueries
}
if len(args.BruteWordList) > 0 {
e.Config.Wordlist = args.BruteWordList
e.Config.Wordlist = args.BruteWordList.Slice()
}
if len(args.AltWordList) > 0 {
e.Config.AltWordlist = args.AltWordList
e.Config.AltWordlist = args.AltWordList.Slice()
}
if len(args.Names) > 0 {
e.ProvidedNames = args.Names
e.ProvidedNames = args.Names.Slice()
}
if args.Options.BruteForcing {
e.Config.BruteForcing = true
Expand All @@ -553,80 +565,43 @@ func updateEnumConfiguration(e *enum.Enumeration, args *enumArgs) error {
e.Config.Passive = true
}
if len(args.Blacklist) > 0 {
e.Config.Blacklist = args.Blacklist
e.Config.Blacklist = args.Blacklist.Slice()
}

disabled := compileDisabledSources(e.GetAllSourceNames(), args.Included, args.Excluded)
if len(disabled) > 0 {
e.Config.DisabledDataSources = disabled
e.Config.DisabledDataSources = disabled.Slice()
}

// Attempt to add the provided domains to the configuration
e.Config.AddDomains(args.Domains)
e.Config.AddDomains(args.Domains.Slice())
if len(e.Config.Domains()) == 0 {
return errors.New("No root domain names were provided")
}
return nil
}

func compileDisabledSources(srcs []string, include, exclude []string) []string {
var inc, disable []string

master := srcs
// Check that the include names are valid
if len(include) > 0 {
for _, incname := range include {
var found bool

for _, name := range master {
if strings.EqualFold(name, incname) {
found = true
inc = append(inc, incname)
break
}
}
func compileDisabledSources(srcs []string, include, exclude stringset.Set) stringset.Set {
master := stringset.New(srcs...)

if !found {
r.Fprintf(color.Error, "%s is not an available data source\n", incname)
}
}
}
// Check that the exclude names are valid
if len(exclude) > 0 {
for _, exclname := range exclude {
var found bool

for _, name := range master {
if strings.EqualFold(name, exclname) {
found = true
disable = append(disable, exclname)
break
}
}

if !found {
r.Fprintf(color.Error, "%s is not an available data source\n", exclname)
}
}
excLen := len(exclude)
exclude.Intersect(master)
if excLen != len(exclude) {
r.Fprintf(color.Error, "Invalid excluded data source specification\n")
}

if len(inc) == 0 {
return disable
// Check that the include names are valid
incLen := len(include)
include.Intersect(master)
if incLen != len(include) {
r.Fprintf(color.Error, "Invalid included data source specification\n")
}
// Data sources missing from the include list are disabled
for _, name := range master {
var found bool

for _, incname := range inc {
if strings.EqualFold(name, incname) {
found = true
break
}
}

if !found {
disable = utils.UniqueAppend(disable, name)
}
if len(include) == 0 {
return exclude
}
return disable

master.Subtract(include)
return master
}
Loading

0 comments on commit d095c1f

Please sign in to comment.