Merge pull request cubefs#1003 from chubaofs/syncFrom_release-20200720

Sync from release-20200720

cli/api/metaapi.go

@@ -87,6 +87,11 @@ type MetaHttpClient struct {
 // NewMasterHelper returns a new MasterClient instance.
 func NewMetaHttpClient(host string, useSSL bool) *MetaHttpClient {
 	mc := &MetaHttpClient{host: host, useSSL: useSSL}
+	var err error
+	_, err = log.InitLog("/tmp/cfs", "cli", log.DebugLevel, nil)
+	if err != nil {
+		fmt.Printf("init cli log err[%v]", err)
+	}
 	return mc
 }
 
@@ -213,6 +218,7 @@ func (mc *MetaHttpClient) GetMetaPartition(pid uint64) (cursor uint64, err error
 		if err != nil {
 			log.LogErrorf("action[GetMetaPartition],pid:%v,err:%v", pid, err)
 		}
+		log.LogFlush()
 	}()
 	request := newAPIRequest(http.MethodGet, "/getPartitionById")
 	request.params["pid"] = fmt.Sprintf("%v", pid)
@@ -236,6 +242,7 @@ func (mc *MetaHttpClient) GetAllDentry(pid uint64) (dentryMap map[string]*metano
 		if err != nil {
 			log.LogErrorf("action[GetAllDentry],pid:%v,err:%v", pid, err)
 		}
+		log.LogFlush()
 	}()
 	dentryMap = make(map[string]*metanode.Dentry, 0)
 	request := newAPIRequest(http.MethodGet, "/getAllDentry")
@@ -286,6 +293,7 @@ func (mc *MetaHttpClient) GetAllInodes(pid uint64) (rstMap map[uint64]*Inode, er
 		if err != nil {
 			log.LogErrorf("action[GetAllInodes],pid:%v,err:%v", pid, err)
 		}
+		log.LogFlush()
 	}()
 	reqURL := fmt.Sprintf("http://%v%v?pid=%v", mc.host, "/getAllInodes", pid)
 	log.LogDebugf("reqURL=%v", reqURL)

cli/build.sh

@@ -1,22 +1,8 @@
 #!/usr/bin/env bash
-RootPath=$(cd $(dirname $0)/..; pwd)
-
-Version=`git describe --abbrev=0 --tags 2>/dev/null`
-BranchName=`git rev-parse --abbrev-ref HEAD 2>/dev/null`
-CommitID=`git rev-parse HEAD 2>/dev/null`
+BranchName=`git rev-parse --abbrev-ref HEAD`
+CommitID=`git rev-parse HEAD`
 BuildTime=`date +%Y-%m-%d\ %H:%M`
 
-SrcPath=${RootPath}/cli
-TargetFile=${1:-$RootPath/cli/cfs-cli}
-
 [[ "-$GOPATH" == "-" ]] && { echo "GOPATH not set"; exit 1; }
 
-LDFlags="-X github.com/chubaofs/chubaofs/proto.Version=${Version} \
-    -X github.com/chubaofs/chubaofs/proto.CommitID=${CommitID} \
-    -X github.com/chubaofs/chubaofs/proto.BranchName=${BranchName} \
-    -X 'github.com/chubaofs/chubaofs/proto.BuildTime=${BuildTime}' "
-
-go build \
-    -ldflags "${LDFlags}" \
-    -o $TargetFile \
-    ${SrcPath}/*.go
+go build -ldflags "-X main.CommitID=${CommitID} -X main.BranchName=${BranchName} -X 'main.BuildTime=${BuildTime}'" -o cfs-cli

cli/cli.go

@@ -16,11 +16,11 @@ package main
 
 import (
 	"fmt"
+	"os"
+
 	"github.com/chubaofs/chubaofs/cli/cmd"
 	"github.com/chubaofs/chubaofs/sdk/master"
-	"github.com/chubaofs/chubaofs/util/log"
 	"github.com/spf13/cobra"
-	"os"
 )
 
 var (
@@ -32,19 +32,17 @@ var (
 func runCLI() (err error) {
 	var cfg *cmd.Config
 	if cfg, err = cmd.LoadConfig(); err != nil {
-		fmt.Printf("init cli log err[%v]", err)
 		return
 	}
-	cfsCli := setupCommands(cfg)
-	if err = cfsCli.Execute(); err != nil {
-		log.LogErrorf("Command fail, err:%v", err)
-	}
+	cfscli := setupCommands(cfg)
+	err = cfscli.Execute()
 	return
 }
 
 func setupCommands(cfg *cmd.Config) *cobra.Command {
 	var mc = master.NewMasterClient(cfg.MasterAddr, false)
-	mc.SetTimeout(cfg.Timeout)
+	mc.DataNodeProfPort = cfg.DataNodeProfPort
+	mc.MetaNodeProfPort = cfg.MetaNodeProfPort
 	cfsRootCmd := cmd.NewRootCmd(mc)
 	var completionCmd = &cobra.Command{
 		Use:   "completion",
@@ -77,10 +75,7 @@ following command to execute:
 
 func main() {
 	var err error
-	_, err = log.InitLog("/tmp/cfs", "cli", log.DebugLevel, nil)
-	defer log.LogFlush()
 	if err = runCLI(); err != nil {
-		log.LogFlush()
 		_, _ = fmt.Fprintf(os.Stderr, "Error: %v\n", err)
 		os.Exit(1)
 	}

cli/cmd/cluster.go

@@ -16,6 +16,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 	"strconv"
 
 	"github.com/chubaofs/chubaofs/proto"
@@ -64,12 +65,14 @@ func newClusterInfoCmd(client *master.MasterClient) *cobra.Command {
 			var cv *proto.ClusterView
 			var delPara map[string]string
 			if cv, err = client.AdminAPI().GetCluster(); err != nil {
-				errout("Error: %v", err)
+				errout("Get cluster info fail:\n%v\n", err)
+				os.Exit(1)
 			}
 			stdout("[Cluster]\n")
 			stdout(formatClusterView(cv))
 			if delPara, err = client.AdminAPI().GetDeleteParas(); err != nil {
-				errout("Error: %v", err)
+				errout("Get delete param fail:\n%v\n", err)
+				os.Exit(1)
 			}
 			stdout(fmt.Sprintf("  BatchCount         : %v\n", delPara[nodeDeleteBatchCountKey]))
 			stdout(fmt.Sprintf("  MarkDeleteRate     : %v\n", delPara[nodeMarkDeleteRateKey]))
@@ -86,18 +89,11 @@ func newClusterStatCmd(client *master.MasterClient) *cobra.Command {
 		Use:   CliOpStatus,
 		Short: cmdClusterStatShort,
 		Run: func(cmd *cobra.Command, args []string) {
-			var (
-				err error
-			    cs  *proto.ClusterStatInfo
-			)
-			defer func() {
-				if err != nil {
-					errout("Error: %v", err)
-				}
-			}()
+			var err error
+			var cs *proto.ClusterStatInfo
 			if cs, err = client.AdminAPI().GetClusterStat(); err != nil {
-				err = fmt.Errorf("Get cluster info fail:\n%v\n", err)
-				return
+				errout("Get cluster info fail:\n%v\n", err)
+				os.Exit(1)
 			}
 			stdout("[Cluster Status]\n")
 			stdout(formatClusterStat(cs))
@@ -109,32 +105,26 @@ func newClusterStatCmd(client *master.MasterClient) *cobra.Command {
 
 func newClusterFreezeCmd(client *master.MasterClient) *cobra.Command {
 	var cmd = &cobra.Command{
-		Use:   CliOpFreeze + " [ENABLE]",
+		Use:       CliOpFreeze + " [ENABLE]",
 		ValidArgs: []string{"true", "false"},
-		Short: cmdClusterFreezeShort,
-		Args:  cobra.MinimumNArgs(1),
+		Short:     cmdClusterFreezeShort,
+		Args:      cobra.MinimumNArgs(1),
 		Long: `Turn on or off the automatic allocation of the data partitions. 
 If 'freeze=false', ChubaoFS WILL automatically allocate new data partitions for the volume when:
   1. the used space is below the max capacity,
   2. and the number of r&w data partition is less than 20.
 		
 If 'freeze=true', ChubaoFS WILL NOT automatically allocate new data partitions `,
 		Run: func(cmd *cobra.Command, args []string) {
-			var (
-				err error
-			    enable bool
-			)
-			defer func() {
-				if err != nil {
-					errout("Error: %v", err)
-				}
-			}()
+			var err error
+			var enable bool
 			if enable, err = strconv.ParseBool(args[0]); err != nil {
-				err = fmt.Errorf("Parse bool fail: %v\n", err)
-				return
+				errout("Parse bool fail: %v\n", err)
+				os.Exit(1)
 			}
 			if err = client.AdminAPI().IsFreezeCluster(enable); err != nil {
-				return
+				errout("Failed: %v\n", err)
+				os.Exit(1)
 			}
 			if enable {
 				stdout("Freeze cluster successful!\n")
@@ -154,25 +144,19 @@ func newClusterSetThresholdCmd(client *master.MasterClient) *cobra.Command {
 		Long: `Set the threshold of memory on each meta node.
 If the memory usage reaches this threshold, all the mata partition will be readOnly.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			var (
-				err error
-			    threshold float64
-			)
-			defer func() {
-				if err != nil {
-					errout("Error: %v", err)
-				}
-			}()
+			var err error
+			var threshold float64
 			if threshold, err = strconv.ParseFloat(args[0], 64); err != nil {
-				err = fmt.Errorf("Parse Float fail: %v\n", err)
-				return
+				errout("Parse Float fail: %v\n", err)
+				os.Exit(1)
 			}
 			if threshold > 1.0 {
-				err = fmt.Errorf("Threshold too big\n")
-				return
+				errout("Threshold too big\n")
+				os.Exit(1)
 			}
 			if err = client.AdminAPI().SetMetaNodeThreshold(threshold); err != nil {
-				return
+				errout("Failed: %v\n", err)
+				os.Exit(1)
 			}
 			stdout("MetaNode threshold is set to %v!\n", threshold)
 		},

cli/cmd/compatibility.go

@@ -15,13 +15,15 @@
 package cmd
 
 import (
-	"fmt"
+	"os"
 	"github.com/chubaofs/chubaofs/cli/api"
+	"github.com/spf13/cobra"
 	"github.com/chubaofs/chubaofs/metanode"
+	"fmt"
+	"strconv"
+	"github.com/chubaofs/chubaofs/util/log"
 	"github.com/chubaofs/chubaofs/proto"
-	"github.com/spf13/cobra"
 	"reflect"
-	"strconv"
 )
 
 const (
@@ -53,21 +55,24 @@ func newMetaCompatibilityCmd() *cobra.Command {
 		Aliases: []string{"meta"},
 		Args:    cobra.MinimumNArgs(3),
 		Run: func(cmd *cobra.Command, args []string) {
+			var err error
 			var (
-				err          error
 				snapshotPath = args[0]
 				host         = args[1]
 				pid          = args[2]
 			)
 			client := api.NewMetaHttpClient(host, false)
 			defer func() {
 				if err != nil {
-					errout("Error: %v", err)
+					errout("Verify metadata consistency failed: %v\n", err)
+					log.LogError(err)
+					log.LogFlush()
+					os.Exit(1)
 				}
 			}()
 			id, err := strconv.ParseUint(pid, 10, 64)
 			if err != nil {
-				err = fmt.Errorf("parse pid[%v] failed: %v\n", pid, err)
+				errout("parse pid[%v] failed: %v\n", pid, err)
 				return
 			}
 			cursor, err := client.GetMetaPartition(id)
@@ -85,9 +90,11 @@ func newMetaCompatibilityCmd() *cobra.Command {
 			}
 			stdout("[Meta partition is %v, verify result]\n", id)
 			if err = verifyDentry(client, mp); err != nil {
+				stdout("%v\n", err)
 				return
 			}
 			if err = verifyInode(client, mp); err != nil {
+				stdout("%v\n", err)
 				return
 			}
 			stdout("All meta has checked\n")
@@ -104,27 +111,25 @@ func verifyDentry(client *api.MetaHttpClient, mp metanode.MetaPartition) (err er
 	mp.GetDentryTree().Ascend(func(d metanode.BtreeItem) bool {
 		dentry, ok := d.(*metanode.Dentry)
 		if !ok {
-			stdout("item type is not *metanode.Dentry \n")
+			stdout("item type is not *metanode.Dentry")
 			err = fmt.Errorf("item type is not *metanode.Dentry")
-			return true
+			return false
 		}
 		key := fmt.Sprintf("%v_%v", dentry.ParentId, dentry.Name)
 		oldDentry, ok := dentryMap[key]
 		if !ok {
-			stdout("dentry %v is not in old version \n", key)
+			stdout("dentry %v is not in old version", key)
 			err = fmt.Errorf("dentry %v is not in old version", key)
 			return false
 		}
 		if !reflect.DeepEqual(dentry, oldDentry) {
-			stdout("dentry %v is not equal with old version \n", key)
+			stdout("dentry %v is not equal with old version", key)
 			err = fmt.Errorf("dentry %v is not equal with old version,dentry[%v],oldDentry[%v]", key, dentry, oldDentry)
 			return false
 		}
 		return true
 	})
-	if err == nil {
-		stdout("The number of dentry is %v, all dentry are consistent \n", mp.GetDentryTree().Len())
-	}
+	stdout("The number of dentry is %v, all dentry are consistent \n", mp.GetDentryTree().Len())
 	return
 }
 
@@ -137,15 +142,12 @@ func verifyInode(client *api.MetaHttpClient, mp metanode.MetaPartition) (err err
 	mp.GetInodeTree().Ascend(func(d metanode.BtreeItem) bool {
 		inode, ok := d.(*metanode.Inode)
 		if !ok {
-			stdout("item type is not *metanode.Inode \n")
-			err = fmt.Errorf("item type is not *metanode.Inode")
 			return true
 		}
 		oldInode, ok := inodesMap[inode.Inode]
 		if !ok {
 			stdout("inode %v is not in old version \n", inode.Inode)
-			err = fmt.Errorf("inode %v is not in old version", inode.Inode)
-			return false
+			return true
 		}
 		localInode = &api.Inode{
 			Inode:      inode.Inode,
@@ -169,13 +171,9 @@ func verifyInode(client *api.MetaHttpClient, mp metanode.MetaPartition) (err err
 		})
 		if !reflect.DeepEqual(oldInode, localInode) {
 			stdout("inode %v is not equal with old version,inode[%v],oldInode[%v]\n", inode.Inode, inode, oldInode)
-			err = fmt.Errorf("inode %v is not equal with old version,inode[%v],oldInode[%v]\n", inode.Inode, inode, oldInode)
-			return false
 		}
 		return true
 	})
-	if err == nil {
-		stdout("The number of inodes is %v, all inodes are consistent \n", mp.GetInodeTree().Len())
-	}
+	stdout("The number of inodes is %v, all inodes are consistent \n", mp.GetInodeTree().Len())
 	return
 }