Feature(provider): set priority from env (metatube-community#48)

engine/engine.go

@@ -3,6 +3,8 @@ package engine
 import (
 	"net/http"
 	"net/url"
+	"os"
+	"strconv"
 	"strings"
 	"time"
 
@@ -15,6 +17,11 @@ import (
 	mt "github.com/metatube-community/metatube-sdk-go/provider"
 )
 
+const (
+	ActorProviderPriorityEnvPrefix = "MT_ACTOR_PROVIDER_PRIORITY_"
+	MovieProviderPriorityEnvPrefix = "MT_MOVIE_PROVIDER_PRIORITY_"
+)
+
 type Engine struct {
 	db      *gorm.DB
 	fetcher *fetch.Fetcher
@@ -37,6 +44,7 @@ func New(db *gorm.DB, timeout time.Duration) *Engine {
 	engine.logger = logger.Sugar()
 	engine.initActorProviders(timeout)
 	engine.initMovieProviders(timeout)
+	engine.initAllProviderPriorities()
 	return engine
 }
 
@@ -50,6 +58,34 @@ func Default() *Engine {
 	return engine
 }
 
+func (e *Engine) initAllProviderPriorities() {
+	for _, env := range os.Environ() {
+		key, value, _ := strings.Cut(strings.ToUpper(env), "=")
+		switch {
+		case strings.HasPrefix(key, ActorProviderPriorityEnvPrefix):
+			name := key[len(ActorProviderPriorityEnvPrefix):]
+			prio, _ := strconv.ParseInt(value, 0, 64)
+			if prio == 0 {
+				delete(e.actorProviders, name)
+				continue
+			}
+			if provider, ok := e.actorProviders[name]; ok {
+				provider.SetPriority(prio)
+			}
+		case strings.HasPrefix(key, MovieProviderPriorityEnvPrefix):
+			name := key[len(MovieProviderPriorityEnvPrefix):]
+			prio, _ := strconv.ParseInt(value, 0, 64)
+			if prio == 0 {
+				delete(e.movieProviders, name)
+				continue
+			}
+			if provider, ok := e.movieProviders[name]; ok {
+				provider.SetPriority(prio)
+			}
+		}
+	}
+}
+
 // initActorProviders initializes actor providers.
 func (e *Engine) initActorProviders(timeout time.Duration) {
 	{ // init

go.mod

@@ -66,7 +66,7 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kennygrant/sanitize v1.2.4 // indirect
-	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.8 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -88,12 +88,12 @@ require (
 	golang.org/x/sync v0.7.0 // indirect
 	golang.org/x/sys v0.21.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.34.1 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/sourcemap.v1 v1.0.5 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	gorm.io/driver/mysql v1.5.6 // indirect
-	modernc.org/libc v1.52.1 // indirect
+	modernc.org/libc v1.53.1 // indirect
 	modernc.org/mathutil v1.6.0 // indirect
 	modernc.org/memory v1.8.0 // indirect
-	modernc.org/sqlite v1.30.0 // indirect
+	modernc.org/sqlite v1.30.1 // indirect
 )

go.sum

@@ -143,8 +143,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm
 github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o=
 github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM=
-github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
+github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM=
+github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
 github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -356,8 +356,8 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
+google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
@@ -381,16 +381,16 @@ gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=
 gorm.io/gorm v1.25.10/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-modernc.org/cc/v4 v4.21.2 h1:dycHFB/jDc3IyacKipCNSDrjIC0Lm1hyoWOZTRR20Lk=
-modernc.org/cc/v4 v4.21.2/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
-modernc.org/ccgo/v4 v4.17.10 h1:6wrtRozgrhCxieCeJh85QsxkX/2FFrT9hdaWPlbn4Zo=
-modernc.org/ccgo/v4 v4.17.10/go.mod h1:0NBHgsqTTpm9cA5z2ccErvGZmtntSM9qD2kFAs6pjXM=
+modernc.org/cc/v4 v4.21.3 h1:2mhBdWKtivdFlLR1ecKXTljPG1mfvbByX7QKztAIJl8=
+modernc.org/cc/v4 v4.21.3/go.mod h1:HM7VJTZbUCR3rV8EYBi9wxnJ0ZBRiGE5OeGXNA0IsLQ=
+modernc.org/ccgo/v4 v4.17.12 h1:0AyNqiL5gJdZr80RN6kQkiuH1jBdq0XMNt+HUXp7NPs=
+modernc.org/ccgo/v4 v4.17.12/go.mod h1:eOo7PuWQu3OfkBH/kYP1/+owPVO+uQAA/XxE3vtKYus=
 modernc.org/fileutil v1.3.0 h1:gQ5SIzK3H9kdfai/5x41oQiKValumqNTDXMvKo62HvE=
 modernc.org/fileutil v1.3.0/go.mod h1:XatxS8fZi3pS8/hKG2GH/ArUogfxjpEKs3Ku3aK4JyQ=
 modernc.org/gc/v2 v2.4.1 h1:9cNzOqPyMJBvrUipmynX0ZohMhcxPtMccYgGOJdOiBw=
 modernc.org/gc/v2 v2.4.1/go.mod h1:wzN5dK1AzVGoH6XOzc3YZ+ey/jPgYHLuVckd62P0GYU=
-modernc.org/libc v1.52.1 h1:uau0VoiT5hnR+SpoWekCKbLqm7v6dhRL3hI+NQhgN3M=
-modernc.org/libc v1.52.1/go.mod h1:HR4nVzFDSDizP620zcMCgjb1/8xk2lg5p/8yjfGv1IQ=
+modernc.org/libc v1.53.1 h1:QuMfWQrOWcgaIaHDCZgUDDECJu5D7wsjLnLDzS7sBAg=
+modernc.org/libc v1.53.1/go.mod h1:RLSlYYmAlXxFAIx9E2UEYh1zJVbocA3oNkq+a1s4jms=
 modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=
 modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo=
 modernc.org/memory v1.8.0 h1:IqGTL6eFMaDZZhEWwcREgeMXYwmW83LYW8cROZYkg+E=
@@ -399,8 +399,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
 modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
 modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
 modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
-modernc.org/sqlite v1.30.0 h1:8YhPUs/HTnlEgErn/jSYQTwHN/ex8CjHHjg+K9iG7LM=
-modernc.org/sqlite v1.30.0/go.mod h1:cgkTARJ9ugeXSNaLBPK3CqbOe7Ec7ZhWPoMFGldEYEw=
+modernc.org/sqlite v1.30.1 h1:YFhPVfu2iIgUf9kuA1CR7iiHdcEEsI2i+yjRYHscyxk=
+modernc.org/sqlite v1.30.1/go.mod h1:DUmsiWQDaAvU4abhc/N+djlom/L2o8f7gZ95RCvyoLU=
 modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
 modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
 modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=

provider/gfriends/gfriends.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/iancoleman/orderedmap"
+	"go.uber.org/atomic"
 
 	"github.com/metatube-community/metatube-sdk-go/common/fetch"
 	"github.com/metatube-community/metatube-sdk-go/common/reverse"
@@ -39,13 +40,17 @@ var (
 	_fetcher = fetch.Default(nil)
 )
 
-type GFriends struct{}
+type GFriends struct {
+	priority *atomic.Int64
+}
 
-func New() *GFriends { return &GFriends{} }
+func New() *GFriends { return &GFriends{atomic.NewInt64(Priority)} }
 
 func (gf *GFriends) Name() string { return Name }
 
-func (gf *GFriends) Priority() int { return Priority }
+func (gf *GFriends) Priority() int64 { return gf.priority.Load() }
+
+func (gf *GFriends) SetPriority(v int64) { gf.priority.Store(v) }
 
 func (gf *GFriends) URL() *url.URL { return _baseURL }
 

provider/internal/scraper/scraper.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/gocolly/colly/v2"
+	"go.uber.org/atomic"
 
 	"github.com/metatube-community/metatube-sdk-go/provider"
 )
@@ -14,7 +15,7 @@ var _ provider.Provider = (*Scraper)(nil)
 // Scraper implements basic Provider interface.
 type Scraper struct {
 	name     string
-	priority int
+	priority *atomic.Int64
 	baseURL  *url.URL
 	c        *colly.Collector
 }
@@ -27,7 +28,7 @@ func NewScraper(name, baseURL string, priority int, opts ...Option) *Scraper {
 	}
 	s := &Scraper{
 		name:     name,
-		priority: priority,
+		priority: atomic.NewInt64(int64(priority)),
 		baseURL:  u,
 		c:        colly.NewCollector(),
 	}
@@ -53,7 +54,9 @@ func (s *Scraper) Name() string { return s.name }
 
 func (s *Scraper) URL() *url.URL { return s.baseURL }
 
-func (s *Scraper) Priority() int { return s.priority }
+func (s *Scraper) Priority() int64 { return s.priority.Load() }
+
+func (s *Scraper) SetPriority(v int64) { s.priority.Store(v) }
 
 func (s *Scraper) NormalizeMovieID(id string) string { return id /* AS IS */ }
 

provider/provider.go

@@ -13,7 +13,10 @@ type Provider interface {
 	Name() string
 
 	// Priority returns the matching priority of the provider.
-	Priority() int
+	Priority() int64
+
+	// SetPriority sets the provider priority to the given value.
+	SetPriority(v int64)
 
 	// URL returns the base url of the provider.
 	URL() *url.URL