Changed --cleanup and --save flag, added caching for orgs/users and n…

…ow org members are skipped by default

.gitignore

@@ -6,3 +6,5 @@ findings.json
 config/rules.json
 config/noisyrules.json
 config/truffleRules.json
+
+notes

robber/analysis.go

@@ -13,6 +13,10 @@ const (
 	B64chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/="
 	// Hexchars is used for entropy finding of hex based strings.
 	Hexchars = "1234567890abcdefABCDEF"
+	// Threshold for b64 matching of entropy strings
+	b64Threshold = 4.5
+	// Threshold for hex matching of entropy strings
+	hexThreshold = 3
 )
 
 // AnalyzeEntropyDiff breaks a given diff into words and finds valid base64 and hex
@@ -23,8 +27,8 @@ func AnalyzeEntropyDiff(m *Middleware, diffObject *DiffObject) {
 	for _, word := range words {
 		b64strings := FindValidStrings(word, B64chars)
 		hexstrings := FindValidStrings(word, Hexchars)
-		PrintEntropyFinding(b64strings, m, diffObject, 4.5)
-		PrintEntropyFinding(hexstrings, m, diffObject, 3)
+		PrintEntropyFinding(b64strings, m, diffObject, b64Threshold)
+		PrintEntropyFinding(hexstrings, m, diffObject, hexThreshold)
 	}
 }
 
@@ -76,7 +80,7 @@ func AnalyzeRepo(m *Middleware, id int, repoch <-chan string, quit chan<- bool,
 				m.Logger.LogFail("Unable to open repo %s: %s\n", reponame, err)
 			}
 
-			commits, err := GetCommits(m.Flags.CommitDepth, repo)
+			commits, err := GetCommits(m, repo, reponame)
 			if err != nil {
 				m.Logger.LogWarn("Unable to fetch commits for %s: %s\n", reponame, err)
 				return

robber/flags.go

@@ -3,10 +3,11 @@ package robber
 import (
 	"errors"
 	"fmt"
-	"github.com/akamensky/argparse"
 	"os"
 	"path/filepath"
 	"strconv"
+
+	"github.com/akamensky/argparse"
 )
 
 const (
@@ -15,22 +16,25 @@ const (
 
 // Flags struct keeps a hold of all of the CLI arguments that were given.
 type Flags struct {
-	Org         *string
-	User        *string
-	Repo        *string
-	Save        *string
-	Config      *os.File
-	Entropy     *bool
-	Both        *bool
-	NoContext   *bool
-	Forks       *bool
-	CleanUp     *bool
-	NoBare      *bool
-	Context     *int
-	CommitDepth *int
-	Noise       *int
-
-	SavePresent bool
+	Org            *string
+	User           *string
+	Repo           *string
+	Save           *string
+	CleanUp        *string
+	Noise          *int
+	Config         *os.File
+	Entropy        *bool
+	Both           *bool
+	NoContext      *bool
+	Forks          *bool
+	NoBare         *bool
+	IncludeMembers *bool
+	NoCache        *bool
+	Context        *int
+	CommitDepth    *int
+
+	SavePresent    bool
+	CleanUpPresent bool
 }
 
 type bound struct {
@@ -49,6 +53,10 @@ func validateInt(argname string, arg string, bound *bound) error {
 	return nil
 }
 
+func validErr(err error) bool {
+	return err.Error() != "not enough arguments for -s|--save" && err.Error() != "not enough arguments for --cleanup"
+}
+
 func flagPresent(shortHand string, name string) bool {
 	for _, val := range os.Args {
 		if val == shortHand || val == name {
@@ -77,11 +85,6 @@ func ParseFlags() *Flags {
 			Help:     "Repository to plunder",
 		}),
 
-		Save: parser.String("s", "save", &argparse.Options{
-			Required: false,
-			Help:     "Yar will save all findings to a specified file",
-		}),
-
 		Context: parser.Int("c", "context", &argparse.Options{
 			Required: false,
 			Help:     "Show N number of lines for context",
@@ -152,24 +155,43 @@ func ParseFlags() *Flags {
 			},
 		}),
 
-		// If cleanup is set, yar will ignore all other flags and only perform cleanup
-		CleanUp: parser.Flag("", "cleanup", &argparse.Options{
+		// Overrides context flag
+		NoContext: parser.Flag("", "no-context", &argparse.Options{
 			Required: false,
-			Help:     "Remove all cloned directories used for caching",
+			Help:     "Only show the secret itself, similar to trufflehog's regex output. Overrides context flag",
 			Default:  false,
 		}),
 
-		// Overrides context flag
-		NoContext: parser.Flag("", "no-context", &argparse.Options{
+		NoCache: parser.Flag("", "--nocache", &argparse.Options{
 			Required: false,
-			Help:     "Only show the secret itself, similar to trufflehog's regex output. Overrides context flag",
+			Help:     "Don't load from cache",
+			Default:  false,
+		}),
+
+		IncludeMembers: parser.Flag("", "--includemembers", &argparse.Options{
+			Required: false,
+			Help:     "Include an organization's members for plunderin'",
 			Default:  false,
 		}),
 
-		SavePresent: flagPresent("-s", "--save"),
+		SavePresent:    flagPresent("-s", "--save"),
+		CleanUpPresent: flagPresent("", "--cleanup"),
+
+		// If cleanup is set, yar will ignore all other flags and only perform cleanup
+		CleanUp: parser.String("", "cleanup", &argparse.Options{
+			Required: false,
+			Help:     "Remove specified cloned directory within yar cache folder. Leave blank to remove the cache folder completely",
+			Default:  "",
+		}),
+
+		Save: parser.String("s", "save", &argparse.Options{
+			Required: false,
+			Help:     "Yar will save all findings to a specified file",
+			Default:  "findings.json",
+		}),
 	}
 
-	if err := parser.Parse(os.Args); err != nil {
+	if err := parser.Parse(os.Args); err != nil && validErr(err) {
 		fmt.Print(parser.Usage(err))
 		os.Exit(1)
 	}
@@ -178,8 +200,11 @@ func ParseFlags() *Flags {
 }
 
 func validateFlags(flags *Flags, parser *argparse.Parser) {
-	if *flags.User == "" && *flags.Repo == "" && *flags.Org == "" && !*flags.CleanUp {
+	if *flags.User == "" && *flags.Repo == "" && *flags.Org == "" && !flags.CleanUpPresent {
 		fmt.Print(parser.Usage("Must give atleast one of org/user/repo"))
 		os.Exit(1)
 	}
+	if *flags.Save == "" {
+		*flags.Save = "findings.json"
+	}
 }

robber/git.go

@@ -79,7 +79,14 @@ func OpenRepo(m *Middleware, path string) (*git.Repository, error) {
 
 // GetCommits simply traverses a given repository, gathering all commits
 // and then returns a list of them.
-func GetCommits(depth *int, repo *git.Repository) ([]*object.Commit, error) {
+func GetCommits(m *Middleware, repo *git.Repository, reponame string) ([]*object.Commit, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			m.Logger.LogFail("%s is corrupted please run yar --cleanup %s and try again\n",
+				reponame[9:], reponame[9:])
+		}
+	}()
+
 	var commits []*object.Commit
 	ref, err := repo.Head()
 	commitIter, err := repo.Log(&git.LogOptions{From: ref.Hash(), Order: git.LogOrderCommitterTime})
@@ -89,7 +96,7 @@ func GetCommits(depth *int, repo *git.Repository) ([]*object.Commit, error) {
 
 	count := 0
 	commitIter.ForEach(func(c *object.Commit) error {
-		if count == *depth {
+		if count == *m.Flags.CommitDepth {
 			return nil
 		}
 		commits = append(commits, c)

robber/github.go

@@ -26,22 +26,50 @@ func handleGithubError(m *Middleware, err error, name string) {
 	m.Logger.LogFail("%s\n", err)
 }
 
+// getCachedUserOrOrg retrieves cached repos under user or org.
+// First tries to read the .git folder under a folder named "name",
+// and if that doesn't exist it assumes that the folder is .git folder
 func getCachedUserOrOrg(name string) []*string {
+	var folderPath string
 	repos := []*string{}
-	folderPath := filepath.Join(os.TempDir(), "yar", name)
+	folderPath = filepath.Join(os.TempDir(), "yar", name)
 	files, err := ioutil.ReadDir(folderPath)
 
 	if err != nil {
-		return []*string{}
+		return repos
 	}
 
 	for _, file := range files {
-		gitFolder := filepath.Join(folderPath, file.Name())
-		repos = append(repos, &gitFolder)
+		if file.Name() == "members.txt" {
+			continue
+		}
+		gitFolder := filepath.Join(folderPath, file.Name(), ".git")
+		if _, err := os.Stat(gitFolder); err != nil {
+			gitFolder = filepath.Dir(gitFolder)
+			repos = append(repos, &gitFolder)
+		} else {
+			repos = append(repos, &gitFolder)
+		}
 	}
 	return repos
 }
 
+func getCachedOrgMembers(orgname string) []*string {
+	members := []*string{}
+	filename := filepath.Join(os.TempDir(), "yar", orgname, "members.txt")
+	content, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return members
+	}
+
+	for _, member := range strings.Split(string(content), "\n") {
+		if member != "" {
+			members = append(members, &member)
+		}
+	}
+	return members
+}
+
 // GetUserRepos returns all non forked public repositories for a given user.
 func GetUserRepos(m *Middleware, username string) []*string {
 	cloneUrls := getCachedUserOrOrg(username)
@@ -71,12 +99,11 @@ func GetUserRepos(m *Middleware, username string) []*string {
 
 // GetOrgRepos returns all repositories of a given organization.
 func GetOrgRepos(m *Middleware, orgname string) []*string {
-	cloneUrls := getCachedUserOrOrg(orgname)
-	if len(cloneUrls) != 0 {
-		return cloneUrls
+	cloneURLs := getCachedUserOrOrg(orgname)
+	if len(cloneURLs) != 0 {
+		return cloneURLs
 	}
 
-	cloneURLs := []*string{}
 	opt := &github.RepositoryListByOrgOptions{ListOptions: github.ListOptions{PerPage: 100}}
 	for {
 		repos, resp, err := m.Client.Repositories.ListByOrg(context.Background(), orgname, opt)
@@ -98,7 +125,11 @@ func GetOrgRepos(m *Middleware, orgname string) []*string {
 
 // GetOrgMembers returns all members of a given organization.
 func GetOrgMembers(m *Middleware, orgname string) []*string {
-	usernames := []*string{}
+	usernames := getCachedOrgMembers(orgname)
+	if len(usernames) != 0 {
+		return usernames
+	}
+
 	opt := &github.ListMembersOptions{ListOptions: github.ListOptions{PerPage: 100}}
 	for {
 		members, resp, err := m.Client.Organizations.ListMembers(context.Background(), orgname, opt)
@@ -112,5 +143,11 @@ func GetOrgMembers(m *Middleware, orgname string) []*string {
 		}
 		opt.Page = resp.NextPage
 	}
+	folderPath := filepath.Join(os.TempDir(), "yar", orgname)
+	os.MkdirAll(folderPath, 0777)
+	err := WriteToFile(filepath.Join(folderPath, "members.txt"), usernames)
+	if err != nil {
+		m.Logger.LogWarn("Failed to save org members of %s due to: %s\n", orgname, err)
+	}
 	return usernames
 }

robber/log.go

@@ -1,7 +1,9 @@
 package robber
 
 import (
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"strings"
 	"sync"
@@ -52,6 +54,18 @@ var logColors = map[int]*color.Color{
 	fail:    color.New(color.FgRed).Add(color.Bold),
 }
 
+type jsonFinding []struct {
+	Reason        string `json:"Reason"`
+	Filepath      string `json:"Filepath"`
+	RepoName      string `json:"RepoName"`
+	Commiter      string `json:"Commiter"`
+	CommitHash    string `json:"CommitHash"`
+	DateOfCommit  string `json:"DateOfCommit"`
+	CommitMessage string `json:"CommitMessage"`
+	Source        string `json:"Source"`
+	Secret        string `json:"Secret"`
+}
+
 // Finding struct contains data of a given secret finding, used for later output of a finding.
 type Finding struct {
 	CommitHash    string
@@ -113,6 +127,35 @@ func NewFinding(reason string, secret []int, diffObject *DiffObject) *Finding {
 	return finding
 }
 
+func saveFindingsHelper(repoName string, hash string, filePath string) string {
+	if strings.HasPrefix(repoName, "/tmp") {
+		return fmt.Sprintf("git --git-dir=%s show %s:%s", repoName, hash[:6], filePath)
+	}
+	return strings.Join([]string{repoName, "commit", hash}, "/")
+}
+
+// SaveFindings saves all findings to a JSON file named findings.json
+func SaveFindings(m *Middleware) {
+	var savedFindings jsonFinding
+	for _, finding := range m.Findings {
+		repoName := strings.Replace(finding.RepoName, ".git", "", 1)
+		source := saveFindingsHelper(repoName, finding.CommitHash, finding.Filepath)
+		savedFindings = append(savedFindings, jsonFinding{{
+			Reason:        finding.Reason,
+			Filepath:      finding.Filepath,
+			RepoName:      repoName,
+			Commiter:      finding.Committer,
+			CommitHash:    finding.CommitHash,
+			DateOfCommit:  finding.DateOfCommit,
+			CommitMessage: finding.CommitMessage,
+			Source:        source,
+			Secret:        finding.Diff[finding.Secret[0]:finding.Secret[1]],
+		}}...)
+	}
+	content, _ := json.MarshalIndent(savedFindings, "", "  ")
+	_ = ioutil.WriteFile(*m.Flags.Save, content, 0644)
+}
+
 func (l *Logger) log(level int, format string, a ...interface{}) {
 	l.Lock()
 	defer l.Unlock()

robber/middleware.go

@@ -34,7 +34,7 @@ func NewMiddleware() *Middleware {
 	}
 	m.Logger = NewLogger(false)
 	// If CleanUp flag is given, handle immediately
-	if *m.Flags.CleanUp {
+	if m.Flags.CleanUpPresent {
 		CleanUp(m)
 	}
 	ParseConfig(m)