Feat: aria2 download and transfer in slave node (cloudreve#1040)

* Feat: retrieve nodes from data table

* Feat: master node ping slave node in REST API

* Feat: master send scheduled ping request

* Feat: inactive nodes recover loop

* Modify: remove database operations from aria2 RPC caller implementation

* Feat: init aria2 client in master node

* Feat: Round Robin load balancer

* Feat: create and monitor aria2 task in master node

* Feat: salve receive and handle heartbeat

* Fix: Node ID will be 0 in download record generated in older version

* Feat: sign request headers with all `X-` prefix

* Feat: API call to slave node will carry meta data in headers

* Feat: call slave aria2 rpc method from master

* Feat: get slave aria2 task status
Feat: encode slave response data using gob

* Feat: aria2 callback to master node / cancel or select task to slave node

* Fix: use dummy aria2 client when caller initialize failed in master node

* Feat: slave aria2 status event callback / salve RPC auth

* Feat: prototype for slave driven filesystem

* Feat: retry for init aria2 client in master node

* Feat: init request client with global options

* Feat: slave receive async task from master

* Fix: competition write in request header

* Refactor: dependency initialize order

* Feat: generic message queue implementation

* Feat: message queue implementation

* Feat: master waiting slave transfer result

* Feat: slave transfer file in stateless policy

* Feat: slave transfer file in slave policy

* Feat: slave transfer file in local policy

* Feat: slave transfer file in OneDrive policy

* Fix: failed to initialize update checker http client

* Feat: list slave nodes for dashboard

* Feat: test aria2 rpc connection in slave

* Feat: add and save node

* Feat: add and delete node in node pool

* Fix: temp file cannot be removed when aria2 task fails

* Fix: delete node in admin panel

* Feat: edit node and get node info

* Modify: delete unused settings

bootstrap/app.go

@@ -34,7 +34,7 @@ type GitHubRelease struct {
 
 // CheckUpdate 检查更新
 func CheckUpdate() {
-	client := request.HTTPClient{}
+	client := request.NewClient()
 	res, err := client.Request("GET", "https://api.github.com/repos/cloudreve/cloudreve/releases", nil).GetResponse()
 	if err != nil {
 		util.Log().Warning("更新检查失败, %s", err)

bootstrap/init.go

@@ -5,9 +5,11 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/aria2"
 	"github.com/cloudreve/Cloudreve/v3/pkg/auth"
 	"github.com/cloudreve/Cloudreve/v3/pkg/cache"
+	"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
 	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/crontab"
 	"github.com/cloudreve/Cloudreve/v3/pkg/email"
+	"github.com/cloudreve/Cloudreve/v3/pkg/slave"
 	"github.com/cloudreve/Cloudreve/v3/pkg/task"
 	"github.com/gin-gonic/gin"
 )
@@ -20,14 +22,86 @@ func Init(path string) {
 	if !conf.SystemConfig.Debug {
 		gin.SetMode(gin.ReleaseMode)
 	}
-	cache.Init()
-	if conf.SystemConfig.Mode == "master" {
-		model.Init()
-		task.Init()
-		aria2.Init(false)
-		email.Init()
-		crontab.Init()
-		InitStatic()
+
+	dependencies := []struct {
+		mode    string
+		factory func()
+	}{
+		{
+			"both",
+			func() {
+				cache.Init()
+			},
+		},
+		{
+			"master",
+			func() {
+				model.Init()
+			},
+		},
+		{
+			"both",
+			func() {
+				task.Init()
+			},
+		},
+		{
+			"master",
+			func() {
+				cluster.Init()
+			},
+		},
+		{
+			"master",
+			func() {
+				aria2.Init(false)
+			},
+		},
+		{
+			"master",
+			func() {
+				email.Init()
+			},
+		},
+		{
+			"master",
+			func() {
+				crontab.Init()
+			},
+		},
+		{
+			"master",
+			func() {
+				InitStatic()
+			},
+		},
+		{
+			"slave",
+			func() {
+				slave.Init()
+			},
+		},
+		{
+			"both",
+			func() {
+				auth.Init()
+			},
+		},
+	}
+
+	for _, dependency := range dependencies {
+		switch dependency.mode {
+		case "master":
+			if conf.SystemConfig.Mode == "master" {
+				dependency.factory()
+			}
+		case "slave":
+			if conf.SystemConfig.Mode == "slave" {
+				dependency.factory()
+			}
+		default:
+			dependency.factory()
+		}
 	}
-	auth.Init()
+
 }

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/gin-gonic/gin v1.5.0
 	github.com/go-ini/ini v1.50.0
 	github.com/go-mail/mail v2.3.1+incompatible
+	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/google/go-querystring v1.0.0
 	github.com/gorilla/websocket v1.4.1

go.sum

@@ -77,6 +77,8 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG
 github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
 github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
+github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=

middleware/auth.go

@@ -22,16 +22,14 @@ import (
 )
 
 // SignRequired 验证请求签名
-func SignRequired() gin.HandlerFunc {
+func SignRequired(authInstance auth.Auth) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		var err error
 		switch c.Request.Method {
-		case "PUT", "POST":
-			err = auth.CheckRequest(auth.General, c.Request)
-			// TODO 生产环境去掉下一行
-			//err = nil
+		case "PUT", "POST", "PATCH":
+			err = auth.CheckRequest(authInstance, c.Request)
 		default:
-			err = auth.CheckURI(auth.General, c.Request.URL)
+			err = auth.CheckURI(authInstance, c.Request.URL)
 		}
 
 		if err != nil {

middleware/auth_test.go

@@ -87,11 +87,10 @@ func TestAuthRequired(t *testing.T) {
 
 func TestSignRequired(t *testing.T) {
 	asserts := assert.New(t)
-	auth.General = auth.HMACAuth{SecretKey: []byte(util.RandStringRunes(256))}
 	rec := httptest.NewRecorder()
 	c, _ := gin.CreateTestContext(rec)
 	c.Request, _ = http.NewRequest("GET", "/test", nil)
-	SignRequiredFunc := SignRequired()
+	SignRequiredFunc := SignRequired(auth.HMACAuth{SecretKey: []byte(util.RandStringRunes(256))})
 
 	// 鉴权失败
 	SignRequiredFunc(c)

middleware/cluster.go

@@ -0,0 +1,62 @@
+package middleware
+
+import (
+	"github.com/cloudreve/Cloudreve/v3/pkg/cluster"
+	"github.com/cloudreve/Cloudreve/v3/pkg/serializer"
+	"github.com/cloudreve/Cloudreve/v3/pkg/slave"
+	"github.com/gin-gonic/gin"
+	"strconv"
+)
+
+// MasterMetadata 解析主机节点发来请求的包含主机节点信息的元数据
+func MasterMetadata() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		c.Set("MasterSiteID", c.GetHeader("X-Site-Id"))
+		c.Set("MasterSiteURL", c.GetHeader("X-Site-Url"))
+		c.Set("MasterVersion", c.GetHeader("X-Cloudreve-Version"))
+		c.Next()
+	}
+}
+
+// UseSlaveAria2Instance 从机用于获取对应主机节点的Aria2实例
+func UseSlaveAria2Instance() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		if siteID, exist := c.Get("MasterSiteID"); exist {
+			// 获取对应主机节点的从机Aria2实例
+			caller, err := slave.DefaultController.GetAria2Instance(siteID.(string))
+			if err != nil {
+				c.JSON(200, serializer.Err(serializer.CodeNotSet, "无法获取 Aria2 实例", err))
+				c.Abort()
+				return
+			}
+
+			c.Set("MasterAria2Instance", caller)
+			c.Next()
+			return
+		}
+
+		c.JSON(200, serializer.ParamErr("未知的主机节点ID", nil))
+		c.Abort()
+	}
+}
+
+func SlaveRPCSignRequired() gin.HandlerFunc {
+	return func(c *gin.Context) {
+		nodeID, err := strconv.ParseUint(c.GetHeader("X-Node-Id"), 10, 64)
+		if err != nil {
+			c.JSON(200, serializer.ParamErr("未知的主机节点ID", err))
+			c.Abort()
+			return
+		}
+
+		slaveNode := cluster.Default.GetNodeByID(uint(nodeID))
+		if slaveNode == nil {
+			c.JSON(200, serializer.ParamErr("未知的主机节点ID", err))
+			c.Abort()
+			return
+		}
+
+		SignRequired(slaveNode.MasterAuthInstance())(c)
+
+	}
+}

models/download.go

@@ -24,6 +24,7 @@ type Download struct {
 	Dst            string `gorm:"type:text"`       // 用户文件系统存储父目录路径
 	UserID         uint   // 发起者UID
 	TaskID         uint   // 对应的转存任务ID
+	NodeID         uint   // 处理任务的节点ID
 
 	// 关联模型
 	User *User `gorm:"PRELOAD:false,association_autoupdate:false"`
@@ -114,3 +115,13 @@ func (task *Download) GetOwner() *User {
 func (download *Download) Delete() error {
 	return DB.Model(download).Delete(download).Error
 }
+
+// GetNodeID 返回任务所属节点ID
+func (task *Download) GetNodeID() uint {
+	// 兼容3.4版本之前生成的下载记录
+	if task.NodeID == 0 {
+		return 1
+	}
+
+	return task.NodeID
+}

models/migration.go

@@ -5,6 +5,7 @@ import (
 	"github.com/cloudreve/Cloudreve/v3/pkg/conf"
 	"github.com/cloudreve/Cloudreve/v3/pkg/util"
 	"github.com/fatih/color"
+	"github.com/gofrs/uuid"
 	"github.com/jinzhu/gorm"
 )
 
@@ -34,8 +35,9 @@ func migration() {
 	if conf.DatabaseConfig.Type == "mysql" {
 		DB = DB.Set("gorm:table_options", "ENGINE=InnoDB")
 	}
+
 	DB.AutoMigrate(&User{}, &Setting{}, &Group{}, &Policy{}, &Folder{}, &File{}, &Share{},
-		&Task{}, &Download{}, &Tag{}, &Webdav{})
+		&Task{}, &Download{}, &Tag{}, &Webdav{}, &Node{})
 
 	// 创建初始存储策略
 	addDefaultPolicy()
@@ -73,6 +75,8 @@ func addDefaultPolicy() {
 }
 
 func addDefaultSettings() {
+	siteID, _ := uuid.NewV4()
+
 	defaultSettings := []Setting{
 		{Name: "siteURL", Value: `http://localhost`, Type: "basic"},
 		{Name: "siteName", Value: `Cloudreve`, Type: "basic"},
@@ -83,6 +87,7 @@ func addDefaultSettings() {
 		{Name: "siteDes", Value: `Cloudreve`, Type: "basic"},
 		{Name: "siteTitle", Value: `平步云端`, Type: "basic"},
 		{Name: "siteScript", Value: ``, Type: "basic"},
+		{Name: "siteID", Value: siteID.String(), Type: "basic"},
 		{Name: "fromName", Value: `Cloudreve`, Type: "mail"},
 		{Name: "mail_keepalive", Value: `30`, Type: "mail"},
 		{Name: "fromAdress", Value: `[email protected]`, Type: "mail"},
@@ -100,10 +105,13 @@ func addDefaultSettings() {
 		{Name: "upload_credential_timeout", Value: `1800`, Type: "timeout"},
 		{Name: "upload_session_timeout", Value: `86400`, Type: "timeout"},
 		{Name: "slave_api_timeout", Value: `60`, Type: "timeout"},
+		{Name: "slave_node_retry", Value: `3`, Type: "slave"},
+		{Name: "slave_ping_interval", Value: `60`, Type: "slave"},
+		{Name: "slave_recover_interval", Value: `120`, Type: "slave"},
+		{Name: "slave_transfer_timeout", Value: `172800`, Type: "timeout"},
 		{Name: "onedrive_monitor_timeout", Value: `600`, Type: "timeout"},
 		{Name: "share_download_session_timeout", Value: `2073600`, Type: "timeout"},
 		{Name: "onedrive_callback_check", Value: `20`, Type: "timeout"},
-		{Name: "aria2_call_timeout", Value: `5`, Type: "timeout"},
 		{Name: "folder_props_timeout", Value: `300`, Type: "timeout"},
 		{Name: "onedrive_chunk_retries", Value: `1`, Type: "retry"},
 		{Name: "onedrive_source_timeout", Value: `1800`, Type: "timeout"},
@@ -131,11 +139,6 @@ Neue',Helvetica,Arial,sans-serif; box-sizing: border-box; font-size: 14px; verti
 		{Name: "gravatar_server", Value: `https://www.gravatar.com/`, Type: "avatar"},
 		{Name: "defaultTheme", Value: `#3f51b5`, Type: "basic"},
 		{Name: "themes", Value: `{"#3f51b5":{"palette":{"primary":{"main":"#3f51b5"},"secondary":{"main":"#f50057"}}},"#2196f3":{"palette":{"primary":{"main":"#2196f3"},"secondary":{"main":"#FFC107"}}},"#673AB7":{"palette":{"primary":{"main":"#673AB7"},"secondary":{"main":"#2196F3"}}},"#E91E63":{"palette":{"primary":{"main":"#E91E63"},"secondary":{"main":"#42A5F5","contrastText":"#fff"}}},"#FF5722":{"palette":{"primary":{"main":"#FF5722"},"secondary":{"main":"#3F51B5"}}},"#FFC107":{"palette":{"primary":{"main":"#FFC107"},"secondary":{"main":"#26C6DA"}}},"#8BC34A":{"palette":{"primary":{"main":"#8BC34A","contrastText":"#fff"},"secondary":{"main":"#FF8A65","contrastText":"#fff"}}},"#009688":{"palette":{"primary":{"main":"#009688"},"secondary":{"main":"#4DD0E1","contrastText":"#fff"}}},"#607D8B":{"palette":{"primary":{"main":"#607D8B"},"secondary":{"main":"#F06292"}}},"#795548":{"palette":{"primary":{"main":"#795548"},"secondary":{"main":"#4CAF50","contrastText":"#fff"}}}}`, Type: "basic"},
-		{Name: "aria2_token", Value: ``, Type: "aria2"},
-		{Name: "aria2_rpcurl", Value: ``, Type: "aria2"},
-		{Name: "aria2_temp_path", Value: ``, Type: "aria2"},
-		{Name: "aria2_options", Value: `{}`, Type: "aria2"},
-		{Name: "aria2_interval", Value: `60`, Type: "aria2"},
 		{Name: "max_worker_num", Value: `10`, Type: "task"},
 		{Name: "max_parallel_transfer", Value: `4`, Type: "task"},
 		{Name: "secret_key", Value: util.RandStringRunes(256), Type: "auth"},