PRT - Lavavisor stability, better UX and resilience to user errors (l…

…avanet#898)

* adjusting lava protocol params

* adding version command

* adding go validation to init

* parallel bootstrap

* adding fetcher capabilities

* adding fetcher to linker tovalidate go path

* adding process manager daemon reload

* adding fetcher to linker + parallel boot

* boot services only once if no version is matching.

* fixing linker issue

* removing binary error to warning

* version monitoring will now trigger even if there is no directory found.

* update on boot.

* script name change to make it more clear

cmd/lavavisor/main.go

@@ -1,24 +1,32 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/cosmos/cosmos-sdk/server"
 	svrcmd "github.com/cosmos/cosmos-sdk/server/cmd"
 	"github.com/lavanet/lava/app"
 	"github.com/lavanet/lava/cmd/lavad/cmd"
 	lvcmd "github.com/lavanet/lava/ecosystem/lavavisor/cmd"
+	"github.com/lavanet/lava/protocol/upgrade"
+	"github.com/spf13/cobra"
 )
 
 func main() {
 	rootCmd := cmd.NewLavaVisorRootCmd()
+
+	// version cobra command
+	cmdVersion := versionCommand()
 	// lavavisor init cobra command
 	cmdLavavisorInit := lvcmd.CreateLavaVisorInitCobraCommand()
 	// lavavisor start cobra command
 	cmdLavavisorStart := lvcmd.CreateLavaVisorStartCobraCommand()
 	// lavavisor start cobra command
 	cmdLavavisorCreateService := lvcmd.CreateLavaVisorCreateServiceCobraCommand()
 
+	// Add Version Command
+	rootCmd.AddCommand(cmdVersion)
 	// Add Lavavisor Init
 	rootCmd.AddCommand(cmdLavavisorInit)
 	// Add Lavavisor Start
@@ -36,3 +44,15 @@ func main() {
 		}
 	}
 }
+
+func versionCommand() *cobra.Command {
+	return &cobra.Command{
+		Use:   "version",
+		Short: "Print the version number",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Print the lavap version
+			version := upgrade.GetCurrentVersion()
+			fmt.Println(version.ProviderVersion) // currently we have only one version.
+		},
+	}
+}

ecosystem/lavavisor/cmd/init.go

@@ -52,7 +52,7 @@ func CreateLavaVisorInitCobraCommand() *cobra.Command {
 
 func LavavisorInit(cmd *cobra.Command) error {
 	dir, _ := cmd.Flags().GetString("directory")
-	lavavisorFetcher := processmanager.ProtocolBinaryFetcher{}
+	lavavisorFetcher := &processmanager.ProtocolBinaryFetcher{}
 
 	// Build path to ./lavavisor
 	err := lavavisorFetcher.SetupLavavisorDir(dir)
@@ -94,7 +94,7 @@ func LavavisorInit(cmd *cobra.Command) error {
 	}
 
 	// linker
-	binaryLinker := processmanager.ProtocolBinaryLinker{}
+	binaryLinker := processmanager.ProtocolBinaryLinker{Fetcher: lavavisorFetcher}
 	err = binaryLinker.CreateLink(binaryPath)
 	if err != nil {
 		return utils.LavaFormatError("Could'nt create link for the protocol binary", err)

ecosystem/lavavisor/cmd/start.go

@@ -3,11 +3,13 @@ package lavavisor
 // TODO: Parallel service restart
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
 	"path/filepath"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/client/flags"
@@ -63,17 +65,44 @@ func (lv *LavaVisor) Start(ctx context.Context, txFactory tx.Factory, clientCtx
 	}
 
 	// Select most recent version set by init command (in the range of min-target version)
-	selectedVersion, err := SelectMostRecentVersionFromDir(lavavisorPath, version.Version)
+	selectedVersion, _ := SelectMostRecentVersionFromDir(lavavisorPath, version.Version)
 	if err != nil {
-		utils.LavaFormatFatal("failed getting most recent version from .lavavisor dir", err)
+		utils.LavaFormatWarning("failed getting most recent version from .lavavisor dir", err)
+	} else {
+		utils.LavaFormatInfo("Version check OK in '.lavavisor' directory.", utils.Attribute{Key: "Selected Version", Value: selectedVersion})
 	}
-	utils.LavaFormatInfo("Version check OK in '.lavavisor' directory.", utils.Attribute{Key: "Selected Version", Value: selectedVersion})
 
 	// Initialize version monitor with selected most recent version
 	versionMonitor := processmanager.NewVersionMonitor(selectedVersion, lavavisorPath, services, autoDownload)
 
 	lavavisorStateTracker.RegisterForVersionUpdates(ctx, version.Version, versionMonitor)
 
+	// check whether lavavisor already started the services when downloading the binaries or not.
+	if !versionMonitor.LaunchedServices {
+		utils.LavaFormatInfo("Version matched existing lavap directory using it to launch the services")
+		// First reload the daemon.
+		err = processmanager.ReloadDaemon()
+		if err != nil {
+			utils.LavaFormatError("Failed reloading daemon", err)
+		}
+		// now start all services
+		var wg sync.WaitGroup
+		for _, process := range services {
+			wg.Add(1)
+			go func(process string) {
+				defer wg.Done() // Decrement the WaitGroup when done
+				utils.LavaFormatInfo("Starting process", utils.Attribute{Key: "Process", Value: process})
+				err := processmanager.StartProcess(process)
+				if err != nil {
+					utils.LavaFormatError("Failed starting process", err, utils.Attribute{Key: "Process", Value: process})
+				}
+			}(process)
+		}
+		// Wait for all Goroutines to finish
+		wg.Wait()
+		utils.LavaFormatInfo("All services launched successfully")
+	}
+
 	// tear down
 	select {
 	case <-ctx.Done():
@@ -106,6 +135,8 @@ func CreateLavaVisorStartCobraCommand() *cobra.Command {
 func LavavisorStart(cmd *cobra.Command) error {
 	dir, _ := cmd.Flags().GetString("directory")
 	binaryFetcher := processmanager.ProtocolBinaryFetcher{}
+	// Validate we have go in Path if we dont we add it to the $PATH and if directory is missing we will download go.
+	binaryFetcher.VerifyGoInstallation()
 	// Build path to ./lavavisor
 	lavavisorPath, err := binaryFetcher.ValidateLavavisorDir(dir)
 	if err != nil {
@@ -146,13 +177,6 @@ func LavavisorStart(cmd *cobra.Command) error {
 	if _, err := os.Stat(lavavisorServicesDir); os.IsNotExist(err) {
 		return utils.LavaFormatError("directory does not exist", nil, utils.Attribute{Key: "lavavisorServicesDir", Value: lavavisorServicesDir})
 	}
-	for _, process := range config.Services {
-		utils.LavaFormatInfo("Starting process", utils.Attribute{Key: "Process", Value: process})
-		err := processmanager.StartProcess(process)
-		if err != nil {
-			utils.LavaFormatError("Failed starting process", err, utils.Attribute{Key: "Process", Value: process})
-		}
-	}
 
 	// Start lavavisor version monitor process
 	lavavisor := LavaVisor{}
@@ -208,7 +232,7 @@ func SelectMostRecentVersionFromDir(lavavisorPath string, version *protocoltypes
 	}
 
 	if selectedVersion == "" {
-		return "", utils.LavaFormatError("No valid version found in the range", nil)
+		return "", fmt.Errorf("did not find any valid versions in lavavisor directory, will try to fetch from github")
 	}
 
 	return selectedVersion, nil

ecosystem/lavavisor/pkg/process/fetcher.go

@@ -6,7 +6,6 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"os/user"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -229,7 +228,7 @@ func (pbf *ProtocolBinaryFetcher) downloadAndBuildFromGithub(version, versionDir
 	utils.LavaFormatInfo("Unzipping...")
 
 	// Verify Go installation
-	goCommand, err := pbf.verifyGoInstallation(pbf.lavavisorPath)
+	goCommand, err := pbf.VerifyGoInstallation()
 	if err != nil {
 		return err
 	}
@@ -316,23 +315,10 @@ func (pbf *ProtocolBinaryFetcher) getInstalledGoVersion(goPath string) (string,
 		utils.LavaFormatError("Unable to parse go version", nil, utils.Attribute{Key: "version", Value: stringGoVersion})
 	}
 	version := versionBeforeCut[2:]
-	utils.LavaFormatDebug("Verified that go is on the right version", utils.Attribute{Key: "version", Value: version})
+	utils.LavaFormatInfo("Verified that go is on the right version", utils.Attribute{Key: "version", Value: version})
 	return version, nil
 }
 
-func (pbf *ProtocolBinaryFetcher) getHomePath() (string, error) {
-	homeDir := os.Getenv("HOME")
-	if homeDir != "" {
-		return homeDir, nil
-	}
-
-	currentUser, err := user.Current()
-	if err != nil {
-		return "", utils.LavaFormatError("Unable to get current user", err)
-	}
-	return currentUser.HomeDir, nil
-}
-
 func (pbf *ProtocolBinaryFetcher) downloadGo(downloadPath string, version string) (string, error) {
 	if runtime.GOARCH == "" {
 		return "", utils.LavaFormatError("Could not determine the machine architecture (runtime.GOARCH is empty). Aborting", nil)
@@ -411,36 +397,38 @@ func (pbf *ProtocolBinaryFetcher) downloadInstallAndVerifyGo(installPath string,
 	return goBinary, nil
 }
 
-func (pbf *ProtocolBinaryFetcher) verifyGoInstallation(lavavisorPath string) (string, error) {
-	emptyGoCommand := ""
+func (pbf *ProtocolBinaryFetcher) VerifyGoInstallation() (string, error) {
 	goCommand := "go"
+	emptyGoCommand := ""
 	expectedGeVersion := "1.20.5"
-
-	homePath, err := pbf.getHomePath()
+	homePath, err := GetHomePath()
 	if err != nil {
 		return emptyGoCommand, err
 	}
-	goPath := filepath.Join(lavavisorPath, "go_installation") // In case go is not installed
+	goPath := filepath.Join(homePath, "go") // In case go is not installed
 
 	installedGoVersion, err := pbf.getInstalledGoVersion(goCommand)
 	if err != nil {
 		utils.LavaFormatInfo("Go was not found in PATH")
-
 		potentialGoCommands := []string{
-			filepath.Join(goPath, "/go/bin/go"),
-			filepath.Join(homePath, "/go/bin/go"), // ~/go/bin/go
-			"/usr/local/go/bin/go",
+			filepath.Join(homePath, "/go/bin"), // ~/go/bin/go
+			"/usr/local/go/bin",
 		}
 
 		found := false
 		for _, potentialGoCommand := range potentialGoCommands {
-			utils.LavaFormatInfo(fmt.Sprintf("Attempting %s", potentialGoCommand))
+			goBin := filepath.Join(potentialGoCommand, "/go")
+			utils.LavaFormatInfo(fmt.Sprintf("Attempting %s", goBin))
 
-			installedGoVersion, err = pbf.getInstalledGoVersion(potentialGoCommand)
+			installedGoVersion, err = pbf.getInstalledGoVersion(goBin)
 			if err == nil {
-				utils.LavaFormatInfo(fmt.Sprintf("Found go %s with version %s", potentialGoCommand, installedGoVersion))
+				utils.LavaFormatInfo(fmt.Sprintf("Found go %s with version %s", goBin, installedGoVersion))
 				found = true
-				goCommand = potentialGoCommand
+				goCommand = goBin
+				err := AddGoPathToDollarPath(potentialGoCommand)
+				if err == nil {
+					return goCommand, nil
+				}
 				break
 			}
 		}

ecosystem/lavavisor/pkg/process/linker.go

@@ -5,11 +5,17 @@ import (
 	"os/exec"
 	"strings"
 
-	lvutil "github.com/lavanet/lava/ecosystem/lavavisor/pkg/util"
 	"github.com/lavanet/lava/utils"
 )
 
-type ProtocolBinaryLinker struct{}
+// TODOs:
+// validate the binary that was created? if our lavap points to old its still bad.
+// on bootstrap just download the right binary.
+// try with which lavap, if it works dont use go path.
+
+type ProtocolBinaryLinker struct {
+	Fetcher *ProtocolBinaryFetcher
+}
 
 func (pbl *ProtocolBinaryLinker) CreateLink(binaryPath string) error {
 	dest, err := pbl.findLavaProtocolPath(binaryPath)
@@ -31,25 +37,18 @@ func (pbl *ProtocolBinaryLinker) findLavaProtocolPath(binaryPath string) (string
 }
 
 func (pbl *ProtocolBinaryLinker) copyBinaryToSystemPath(binaryPath string) (string, error) {
-	gobin, err := exec.Command("go", "env", "GOPATH").Output()
+	goPath, err := pbl.Fetcher.VerifyGoInstallation()
 	if err != nil {
-		return "", utils.LavaFormatError("Couldn't determine Go binary path", err)
+		return "", utils.LavaFormatError("Couldn't get go binary path", err)
 	}
-
-	goBinPath := strings.TrimSpace(string(gobin)) + "/bin/"
-	pbl.validateBinaryExecutable(binaryPath)
-	pbl.removeExistingLink(goBinPath + "lavap")
-
-	err = lvutil.Copy(binaryPath, goBinPath+"lavap")
+	goBin, err := exec.Command(goPath, "env", "GOPATH").Output()
 	if err != nil {
-		return "", utils.LavaFormatError("Couldn't copy binary to system path", err)
+		return "", utils.LavaFormatError("Couldn't determine Go binary path", err)
 	}
 
-	out, err := exec.LookPath("lavap")
-	if err != nil {
-		return "", utils.LavaFormatError("Couldn't find the binary in the system path", err)
-	}
-	return strings.TrimSpace(out), nil
+	goBinPath := strings.TrimSpace(string(goBin)) + "/bin/"
+	pbl.validateBinaryExecutable(binaryPath)
+	return goBinPath + "lavap", nil
 }
 
 func (pbl *ProtocolBinaryLinker) validateBinaryExecutable(path string) {
@@ -69,6 +68,7 @@ func (pbl *ProtocolBinaryLinker) removeExistingLink(linkPath string) {
 	} else if !os.IsNotExist(err) {
 		utils.LavaFormatFatal("Unexpected error when checking for existing link", err)
 	}
+	utils.LavaFormatInfo("Removed Link Successfully")
 }
 
 func (pbl *ProtocolBinaryLinker) createAndVerifySymlink(binaryPath, dest string) {

ecosystem/lavavisor/pkg/process/process_manager.go

@@ -1,17 +1,24 @@
 package processmanager
 
 import (
+	"fmt"
+	"os"
 	"os/exec"
+	"os/user"
 	"strings"
 
 	protocolVersion "github.com/lavanet/lava/protocol/upgrade"
 	"github.com/lavanet/lava/utils"
 	protocoltypes "github.com/lavanet/lava/x/protocol/types"
 )
 
-type ServiceProcess struct {
-	Name    string
-	ChainID string
+func ReloadDaemon() error {
+	cmd := exec.Command("sudo", "systemctl", "daemon-reload")
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return utils.LavaFormatError("Failed to run command", err, utils.Attribute{Key: "Output", Value: output})
+	}
+	return nil
 }
 
 func StartProcess(process string) error {
@@ -20,7 +27,6 @@ func StartProcess(process string) error {
 
 	// Create command list
 	cmds := []*exec.Cmd{
-		exec.Command("sudo", "systemctl", "daemon-reload"),
 		exec.Command("sudo", "systemctl", "enable", process),
 		exec.Command("sudo", "systemctl", "restart", process),
 		exec.Command("sudo", "systemctl", "status", process),
@@ -45,7 +51,7 @@ func GetBinaryVersion(binaryPath string) (string, error) {
 	cmd := exec.Command(binaryPath, "version")
 	output, err := cmd.Output()
 	if err != nil {
-		return "", utils.LavaFormatError("GetBinaryVersion failed to execute command", err)
+		return "", utils.LavaFormatWarning("GetBinaryVersion failed to execute command, lavavisor will try to fetch version from github", err)
 	}
 	return strings.TrimSpace(string(output)), nil
 }
@@ -56,3 +62,32 @@ func ValidateMismatch(incoming *protocoltypes.Version, current string) bool {
 		protocolVersion.HasVersionMismatch(incoming.ConsumerTarget, current) ||
 		protocolVersion.HasVersionMismatch(incoming.ProviderTarget, current))
 }
+
+func GetHomePath() (string, error) {
+	homeDir := os.Getenv("HOME")
+	if homeDir != "" {
+		return homeDir, nil
+	}
+
+	currentUser, err := user.Current()
+	if err != nil {
+		return "", utils.LavaFormatError("Unable to get current user", err)
+	}
+	return currentUser.HomeDir, nil
+}
+
+func AddGoPathToDollarPath(path string) error {
+	// Get the current PATH
+	currentPath := os.Getenv("PATH")
+	// Check if the default Go bin path is already in the PATH
+	if strings.Contains(currentPath, path) {
+		utils.LavaFormatInfo("Validation completed successfully - Default Go bin path already exists in PATH")
+		return nil
+	}
+	utils.LavaFormatInfo("Adding Path to $PATH", utils.Attribute{Key: "path", Value: path})
+	// Append the default Go bin path to the existing PATH
+	newPath := fmt.Sprintf("%s:%s", currentPath, path)
+	// Set the updated PATH
+	err := os.Setenv("PATH", newPath)
+	return err
+}