Adding support for both Yml and yaml extensions (#18)

* add support for both yml and yaml extensions

* revert build command changes

* replace with build commit

* fix typos and reduce cyclomatic complexity

* get file name before running test discovery

build/nucleus/build.sh

@@ -9,4 +9,4 @@ trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT
 
 echo 'Building binary'
 go build -o nucleus ./cmd/nucleus/*.go
-echo 'Binary succesfully build by the name of `nucleus`'
+echo 'Binary successfully build by the name of `nucleus`'

build/synapse/build.sh

@@ -9,4 +9,4 @@ trap 'echo "\"${last_command}\" command filed with exit code $?."' EXIT
 
 echo 'Building binary'
 go build -o synapse ./cmd/synapse/*.go
-echo 'Binary succesfully build by the name of `synapse`'
+echo 'Binary successfully build by the name of `synapse`'

cmd/nucleus/bin.go

@@ -139,10 +139,8 @@ func run(cmd *cobra.Command, args []string) {
 		logger.Fatalf("failed to initialize cache manager: %v", err)
 	}
 
-	parserService, err := parser.New(ctx, tcm, logger)
-	if err != nil {
-		logger.Fatalf("failed to initialize parser service: %v", err)
-	}
+	parserService := parser.New(tcm, logger)
+
 	coverageService, err := coverage.New(execManager, azureClient, zstd, cfg, logger)
 	if err != nil {
 		logger.Fatalf("failed to initialize coverage service: %v", err)

go.mod

@@ -5,13 +5,14 @@ go 1.17
 require (
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.14.0
+	github.com/bmatcuk/doublestar/v4 v4.0.2
 	github.com/coreos/go-semver v0.3.0
+	github.com/denisbrodbeck/machineid v1.0.1
 	github.com/docker/docker v20.10.12+incompatible
 	github.com/gin-gonic/gin v1.7.7
 	github.com/go-playground/locales v0.14.0
 	github.com/go-playground/universal-translator v0.18.0
 	github.com/go-playground/validator/v10 v10.10.0
-	github.com/google/uuid v1.2.0
 	github.com/gorilla/websocket v1.4.2
 	github.com/joho/godotenv v1.4.0
 	github.com/mholt/archiver/v3 v3.5.1
@@ -32,7 +33,6 @@ require (
 	github.com/andybalholm/brotli v1.0.1 // indirect
 	github.com/containerd/containerd v1.5.9 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/denisbrodbeck/machineid v1.0.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
@@ -43,6 +43,7 @@ require (
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
+	github.com/google/uuid v1.2.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect

go.sum

@@ -130,6 +130,8 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE
 github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/bmatcuk/doublestar/v4 v4.0.2 h1:X0krlUVAVmtr2cRoTqR8aDMrDqnB36ht8wpWTiQ3jsA=
+github.com/bmatcuk/doublestar/v4 v4.0.2/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=

pkg/core/interfaces.go

@@ -23,8 +23,6 @@ type TASConfigManager interface {
 type GitManager interface {
 	// Clone repository from TAS config
 	Clone(ctx context.Context, payload *Payload, cloneToken string) error
-	// CloneYML  clones all .tas.yml for all  the commits
-	CloneYML(ctx context.Context, payload *Payload, cloneToken string) error
 }
 
 // DiffManager manages the diff findings for the given payload
@@ -54,9 +52,10 @@ type CoverageService interface {
 	MergeAndUpload(ctx context.Context, payload *Payload) error
 }
 
-// YMLParserService services parsing of tas.yml
+// YMLParserService parses the .tas.yml files
 type YMLParserService interface {
-	PerformParsing(payload *Payload) error
+	// ParseAndValidate the YML file and validades it
+	ParseAndValidate(ctx context.Context, payload *Payload) error
 }
 
 // TestStats is used for servicing stat collection

pkg/core/lifecycle.go

@@ -78,11 +78,11 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 	// set payload on pipeline object
 	pl.Payload = payload
 	if pl.Cfg.ParseMode {
-		err = pl.GitManager.CloneYML(ctx, payload, oauth.Data.AccessToken)
+		err = pl.GitManager.Clone(ctx, payload, oauth.Data.AccessToken)
 		if err != nil {
 			pl.Logger.Fatalf("failed to clone YML for build ID: %s, error: %v", payload.BuildID, err)
 		}
-		if err = pl.ParserService.PerformParsing(payload); err != nil {
+		if err = pl.ParserService.ParseAndValidate(ctx, payload); err != nil {
 			pl.Logger.Fatalf("error while parsing YML for build ID: %s, error: %v", payload.BuildID, err)
 		}
 		os.Exit(0)
@@ -117,7 +117,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 		if p := recover(); p != nil {
 			pl.Logger.Errorf("panic stack trace: %v", p)
 			taskPayload.Status = Error
-			taskPayload.Remark = errs.GenericUserFacingBEErrRemark
+			taskPayload.Remark = errs.GenericErrRemark.Error()
 		} else if err != nil {
 			if err == context.Canceled {
 				taskPayload.Status = Aborted
@@ -180,7 +180,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 		err = pl.ExecutionManager.ExecuteInternalCommands(ctx, InstallNodeVer, command, "", nil, nil)
 		if err != nil {
 			pl.Logger.Errorf("Unable to install user-defined nodeversion %v", err)
-			errRemark = errs.GenericUserFacingBEErrRemark
+			errRemark = errs.GenericErrRemark.Error()
 			return err
 		}
 		origPath := os.Getenv("PATH")
@@ -190,31 +190,31 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 	if payload.CollectCoverage {
 		if err = fileutils.CreateIfNotExists(coverageDir, true); err != nil {
 			pl.Logger.Errorf("failed to create coverage directory %v", err)
-			errRemark = errs.GenericUserFacingBEErrRemark
+			errRemark = errs.GenericErrRemark.Error()
 			return err
 		}
 	}
 
 	err = pl.TestBlockListService.GetBlockListedTests(ctx, tasConfig, payload.RepoID)
 	if err != nil {
 		pl.Logger.Errorf("Unable to fetch blocklisted tests: %v", err)
-		errRemark = errs.GenericUserFacingBEErrRemark
+		errRemark = errs.GenericErrRemark.Error()
 		return err
 	}
 
 	// read secrets
 	secretMap, err := pl.SecretParser.GetRepoSecret(global.RepoSecretPath)
 	if err != nil {
 		pl.Logger.Errorf("Error in fetching Repo secrets %v", err)
-		errRemark = errs.GenericUserFacingBEErrRemark
+		errRemark = errs.GenericErrRemark.Error()
 		return err
 	}
 
 	cacheKey := fmt.Sprintf("%s/%s/%s", payload.OrgID, payload.RepoID, tasConfig.Cache.Key)
 	// TODO:  download from cdn
 	if err = pl.CacheStore.Download(ctx, cacheKey); err != nil {
 		pl.Logger.Errorf("Unable to download cache: %v", err)
-		errRemark = errs.GenericUserFacingBEErrRemark
+		errRemark = errs.GenericErrRemark.Error()
 		return err
 	}
 
@@ -230,7 +230,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 	err = pl.ExecutionManager.ExecuteInternalCommands(ctx, InstallRunners, global.InstallRunnerCmd, global.RepoDir, nil, nil)
 	if err != nil {
 		pl.Logger.Errorf("Unable to install custom runners %v", err)
-		errRemark = errs.GenericUserFacingBEErrRemark
+		errRemark = errs.GenericErrRemark.Error()
 		return err
 	}
 
@@ -266,7 +266,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 
 		if err = pl.sendStats(*executionResult); err != nil {
 			pl.Logger.Errorf("error while sending test reports %v", err)
-			errRemark = errs.GenericUserFacingBEErrRemark
+			errRemark = errs.GenericErrRemark.Error()
 			return err
 		}
 		taskPayload.Status = Passed
@@ -290,7 +290,7 @@ func (pl *Pipeline) Start(ctx context.Context) (err error) {
 	}
 	if err = pl.CacheStore.Upload(ctx, cacheKey, tasConfig.Cache.Paths...); err != nil {
 		pl.Logger.Errorf("Unable to upload cache: %v", err)
-		errRemark = errs.GenericUserFacingBEErrRemark
+		errRemark = errs.GenericErrRemark.Error()
 		return err
 	}
 	pl.Logger.Debugf("Cache uploaded successfully")

pkg/core/models.go

@@ -238,8 +238,8 @@ type TaskPayload struct {
 	Type        TaskType  `json:"type"`
 }
 
-//CoverageMainfest for post processing coverage job
-type CoverageMainfest struct {
+// CoverageManifest for post processing coverage job
+type CoverageManifest struct {
 	Removedfiles      []string           `json:"removed_files"`
 	AllFilesExecuted  bool               `json:"all_files_executed"`
 	CoverageThreshold *CoverageThreshold `json:"coverage_threshold,omitempty"`
@@ -270,7 +270,7 @@ type Oauth struct {
 	} `json:"data"`
 }
 
-//TASConfig represents the .tas.yml file
+// TASConfig represents the .tas.yml file
 type TASConfig struct {
 	SmartRun          bool               `yaml:"smartRun"`
 	Framework         string             `yaml:"framework" validate:"required,oneof=jest mocha jasmine"`

pkg/core/runner.go

@@ -29,7 +29,7 @@ type ContainerStatus struct {
 	Error errs.Err
 }
 
-// ContainerImageConfig contains config which contian registery for docker
+// ContainerImageConfig contains config which contains the registry for docker
 type ContainerImageConfig struct {
 	AuthRegistry string
 	Image        string

pkg/errs/nucleus.go

@@ -4,17 +4,13 @@ import (
 	"fmt"
 )
 
-// GenericUserFacingBEErrRemark returns a generic error message for user facing errors.
-const GenericUserFacingBEErrRemark = "Unexpected error"
-
-// Err repersent structure of error
+// Err represents structure of a custom error
 type Err struct {
 	Code    string
 	Message string
 	URL     string
 }
 
-// TODO: Print err url
 func (e Err) Error() string {
 	return fmt.Sprintf("%s : %s ", e.Code, e.Message)
 }
@@ -63,4 +59,6 @@ var (
 	ErrUnsupportedGitProvider = New("unsupported gitprovider")
 	// ErrGitDiffNotFound is returned when basecommit is null or git provider returns empty diff
 	ErrGitDiffNotFound = New("diff not found")
+	// GenericErrRemark returns a generic error message for user facing errors.
+	GenericErrRemark = New("Unexpected error")
 )

pkg/errs/synapse.go

@@ -227,7 +227,7 @@ var ERR_SNK_RD_WRT_MSM = Err{
 	Code:    "ERR::SNK::RD::WRT::MSM",
 	Message: "Read write mismatch in sink proxy "}
 
-// CR_AUTH_NF should be raise when contianer registry auth are not present for private repo
+// CR_AUTH_NF should be raise when container registry auth are not present for private repo
 var CR_AUTH_NF = Err{
 	Code:    "CR::AUTH:NF",
-	Message: "Contianer registry auth are not present for private repo"}
+	Message: "Container registry auth are not present for private repo"}

pkg/gitmanager/setup.go

@@ -34,7 +34,7 @@ func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, cloneTok
 	repoLink := payload.RepoLink
 	repoItems := strings.Split(repoLink, "/")
 	repoName := repoItems[len(repoItems)-1]
-	commitID := payload.TargetCommit
+	commitID := payload.BuildTargetCommit
 	archiveURL, err := urlmanager.GetCloneURL(payload.GitProvider, repoLink, repoName, commitID)
 	if err != nil {
 		gm.logger.Errorf("failed to get clone url for provider %s, error %v", payload.GitProvider, err)
@@ -55,31 +55,6 @@ func (gm *gitManager) Clone(ctx context.Context, payload *core.Payload, cloneTok
 	return nil
 }
 
-func (gm *gitManager) CloneYML(ctx context.Context, payload *core.Payload, cloneToken string) error {
-	if err := os.Mkdir(global.RepoDir, os.ModePerm); err != nil {
-		gm.logger.Errorf("failed to create dir %s, error: %v", global.RepoDir, err)
-		return err
-	}
-
-	commitID := payload.BuildTargetCommit
-	archiveURL, err := urlmanager.GetDownloadURL(payload.GitProvider, payload.RepoSlug, commitID, payload.TasFileName)
-	if err != nil {
-		gm.logger.Errorf("failed to get download url for provider %s, error %v", payload.GitProvider, err)
-		return err
-	}
-	tasConfigFilePath := commitID + payload.TasFileName
-	if err := gm.downloadFile(ctx, archiveURL, commitID+payload.TasFileName, cloneToken); err != nil {
-		gm.logger.Errorf("error while cloning yaml for commitID %s, error: %v", commitID, err)
-		return err
-	}
-	gm.logger.Debugf("downloaded yaml file %s", tasConfigFilePath)
-	if err := os.Rename(tasConfigFilePath, filepath.Join(global.RepoDir, tasConfigFilePath)); err != nil {
-		gm.logger.Errorf("failed to move dir commitID %s, error: %v", err)
-		return err
-	}
-	return nil
-}
-
 // downloadFile clones the archive from github and extracts the file if it is a zip file.
 func (gm *gitManager) downloadFile(ctx context.Context, archiveURL, fileName, cloneToken string) error {
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, archiveURL, nil)

pkg/runner/docker/docker.go

@@ -167,7 +167,7 @@ func (d *docker) waitForRunning(ctx context.Context, r *core.RunnerOptions) erro
 	case status := <-statusCh:
 		d.logger.Debugf("status code: %d", status.StatusCode)
 		if status.StatusCode != 0 {
-			msg := fmt.Sprintf("Recieved non zero status code %v", status.StatusCode)
+			msg := fmt.Sprintf("Received non zero status code %v", status.StatusCode)
 			return errs.ERR_DOCKER_RUN(msg)
 
 		}
@@ -195,7 +195,7 @@ func (d *docker) Initiate(ctx context.Context, r *core.RunnerOptions, statusChan
 		statusChan <- status
 		return
 	}
-	d.logger.Infof("container %+s executuion succesful", r.ContainerID)
+	d.logger.Infof("container %+s executuion successful", r.ContainerID)
 	statusChan <- core.ContainerStatus{Done: true}
 }
 

pkg/service/coverage/coverage.go

@@ -29,7 +29,7 @@ const (
 	coverageJSONFileName = "coverage-final.json"
 	mergedcoverageJSON   = "coverage-merged.json"
 	compressedFileName   = "coverage-files.tzst"
-	mainfestJSONFileName = "manifest.json"
+	manifestJSONFileName = "manifest.json"
 	coverageFilePath     = "/scripts/mapCoverage.js"
 )
 
@@ -129,7 +129,7 @@ func (c *codeCoverageService) MergeAndUpload(ctx context.Context, payload *core.
 			c.logger.Errorf("code coverage directory not found commit id %s", commit.Sha)
 			return err
 		}
-		coverageManifestPath := filepath.Join(commitDir, mainfestJSONFileName)
+		coverageManifestPath := filepath.Join(commitDir, manifestJSONFileName)
 
 		manifestPayload, err := c.parseManifestFile(coverageManifestPath)
 		if err != nil {
@@ -159,28 +159,19 @@ func (c *codeCoverageService) MergeAndUpload(ctx context.Context, payload *core.
 				return err
 			}
 			_, err := c.uploadFile(ctx, repoBlobPath, compressedFileName, commit.Sha)
-			if err != nil {
-				return err
-			}
-			return nil
+			return err
 		})
 
 		var blobURL string
 		g.Go(func() error {
 			blobURL, err = c.uploadFile(ctx, repoBlobPath, filepath.Join(commitDir, mergedcoverageJSON), commit.Sha)
-			if err != nil {
-				return err
-			}
-			return nil
+			return err
 		})
 
 		var totalCoverage json.RawMessage
 		g.Go(func() error {
 			totalCoverage, err = c.getTotalCoverage(filepath.Join(commitDir, mergedcoverageJSON))
-			if err != nil {
-				return err
-			}
-			return nil
+			return err
 		})
 		if err = g.Wait(); err != nil {
 			c.logger.Errorf("failed to upload files to azure blob %v", err)
@@ -209,8 +200,8 @@ func (c *codeCoverageService) uploadFile(ctx context.Context, blobPath, filename
 	return
 }
 
-func (c *codeCoverageService) parseManifestFile(filepath string) (core.CoverageMainfest, error) {
-	manifestPayload := core.CoverageMainfest{}
+func (c *codeCoverageService) parseManifestFile(filepath string) (core.CoverageManifest, error) {
+	manifestPayload := core.CoverageManifest{}
 	if _, err := os.Stat(filepath); os.IsNotExist(err) {
 		c.logger.Errorf("manifest file not found in path %s", filepath)
 		return manifestPayload, err