Add per feed cookies option

Launacloud · Mar 23, 2021 · 9242350 · 9242350
1 parent b0c14aa
commit 9242350
Show file tree

Hide file tree

Showing 34 changed files with 114 additions and 15 deletions.
diff --git a/api/subscription.go b/api/subscription.go
@@ -29,6 +29,7 @@ func (h *handler) discoverSubscriptions(w http.ResponseWriter, r *http.Request)
 	subscriptions, finderErr := subscription.FindSubscriptions(
 		subscriptionDiscoveryRequest.URL,
 		subscriptionDiscoveryRequest.UserAgent,
+		subscriptionDiscoveryRequest.Cookie,
 		subscriptionDiscoveryRequest.Username,
 		subscriptionDiscoveryRequest.Password,
 		subscriptionDiscoveryRequest.FetchViaProxy,

diff --git a/client/model.go b/client/model.go
@@ -122,6 +122,7 @@ type Feed struct {
 	KeeplistRules               string    `json:"keeplist_rules"`
 	Crawler                     bool      `json:"crawler"`
 	UserAgent                   string    `json:"user_agent"`
+	Cookie                      string    `json:"cookie"`
 	Username                    string    `json:"username"`
 	Password                    string    `json:"password"`
 	Category                    *Category `json:"category,omitempty"`
@@ -132,6 +133,7 @@ type FeedCreationRequest struct {
 	FeedURL                     string `json:"feed_url"`
 	CategoryID                  int64  `json:"category_id"`
 	UserAgent                   string `json:"user_agent"`
+	Cookie                      string `json:"cookie"`
 	Username                    string `json:"username"`
 	Password                    string `json:"password"`
 	Crawler                     bool   `json:"crawler"`
@@ -156,6 +158,7 @@ type FeedModificationRequest struct {
 	KeeplistRules               *string `json:"keeplist_rules"`
 	Crawler                     *bool   `json:"crawler"`
 	UserAgent                   *string `json:"user_agent"`
+	Cookie                      *string `json:"cookie"`
 	Username                    *string `json:"username"`
 	Password                    *string `json:"password"`
 	CategoryID                  *int64  `json:"category_id"`

diff --git a/database/migrations.go b/database/migrations.go
@@ -529,4 +529,9 @@ var migrations = []func(tx *sql.Tx) error{
 		_, err = tx.Exec(sql)
 		return err
 	},
+	func(tx *sql.Tx) (err error) {
+		sql := `ALTER TABLE feeds ADD COLUMN cookie text default ''`
+		_, err = tx.Exec(sql)
+		return err
+	},
 }
diff --git a/http/client/client.go b/http/client/client.go
@@ -47,6 +47,7 @@ type Client struct {
 	requestUsername            string
 	requestPassword            string
 	requestUserAgent           string
+	requestCookie              string
 
 	useProxy             bool
 	doNotFollowRedirects bool
@@ -142,6 +143,14 @@ func (c *Client) WithUserAgent(userAgent string) *Client {
 	return c
 }
 
+// WithCookie defines the Cookies to use for HTTP requests.
+func (c *Client) WithCookie(cookie string) *Client {
+	if cookie != "" {
+		c.requestCookie = cookie
+	}
+	return c
+}
+
 // Get performs a GET HTTP request.
 func (c *Client) Get() (*Response, error) {
 	request, err := c.buildRequest(http.MethodGet, nil)
@@ -336,6 +345,10 @@ func (c *Client) buildHeaders() http.Header {
 		headers.Add("Authorization", c.requestAuthorizationHeader)
 	}
 
+	if c.requestCookie != "" {
+		headers.Add("Cookie", c.requestCookie)
+	}
+
 	headers.Add("Connection", "close")
 	return headers
 }
diff --git a/locale/translations/de_DE.json b/locale/translations/de_DE.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Benutzername des Abonnements",
     "form.feed.label.feed_password": "Passwort des Abonnements",
     "form.feed.label.user_agent": "Standardbenutzeragenten überschreiben",
+    "form.feed.label.cookie": "Cookies setzten",
     "form.feed.label.scraper_rules": "Extraktionsregeln",
     "form.feed.label.rewrite_rules": "Umschreiberegeln",
     "form.feed.label.blocklist_rules": "Blockierregeln",

diff --git a/locale/translations/en_US.json b/locale/translations/en_US.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Feed Username",
     "form.feed.label.feed_password": "Feed Password",
     "form.feed.label.user_agent": "Override Default User Agent",
+    "form.feed.label.cookie": "Set Cookies",
     "form.feed.label.scraper_rules": "Scraper Rules",
     "form.feed.label.rewrite_rules": "Rewrite Rules",
     "form.feed.label.blocklist_rules": "Block Rules",

diff --git a/locale/translations/es_ES.json b/locale/translations/es_ES.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Nombre de usuario de fuente",
     "form.feed.label.feed_password": "Contraseña de fuente",
     "form.feed.label.user_agent": "Invalidar el agente de usuario predeterminado",
+    "form.feed.label.cookie": "Configurar las cookies",
     "form.feed.label.scraper_rules": "Reglas de raspador",
     "form.feed.label.rewrite_rules": "Reglas de reescribir",
     "form.feed.label.blocklist_rules": "Reglas de Filtrado(Bloquear)",

diff --git a/locale/translations/fr_FR.json b/locale/translations/fr_FR.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Nom d'utilisateur du flux",
     "form.feed.label.feed_password": "Mot de passe du flux",
     "form.feed.label.user_agent": "Remplacer l'agent utilisateur par défaut",
+    "form.feed.label.cookie": "Définir les cookies",
     "form.feed.label.scraper_rules": "Règles pour récupérer le contenu original",
     "form.feed.label.rewrite_rules": "Règles de réécriture",
     "form.feed.label.blocklist_rules": "Règles de blocage",

diff --git a/locale/translations/it_IT.json b/locale/translations/it_IT.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Nome utente del feed",
     "form.feed.label.feed_password": "Password del feed",
     "form.feed.label.user_agent": "Usa user agent personalizzato",
+    "form.feed.label.cookie": "Installare i cookies",
     "form.feed.label.scraper_rules": "Regole di estrazione del contenuto",
     "form.feed.label.rewrite_rules": "Regole di impaginazione del contenuto",
     "form.feed.label.blocklist_rules": "Regole di blocco",

diff --git a/locale/translations/ja_JP.json b/locale/translations/ja_JP.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "フィードのユーザー名",
     "form.feed.label.feed_password": "フィードのパスワード",
     "form.feed.label.user_agent": "ディフォルトの User Agent を上書きする",
+    "form.feed.label.cookie": "クッキーの設定",
     "form.feed.label.scraper_rules": "スクラップルール",
     "form.feed.label.rewrite_rules": "Rewrite ルール",
     "form.feed.label.blocklist_rules": "ブロックルール",

diff --git a/locale/translations/nl_NL.json b/locale/translations/nl_NL.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Feed-gebruikersnaam",
     "form.feed.label.feed_password": "Feed wachtwoord",
     "form.feed.label.user_agent": "Standaard User Agent overschrijven",
+    "form.feed.label.cookie": "Cookies instellen",
     "form.feed.label.scraper_rules": "Scraper regels",
     "form.feed.label.rewrite_rules": "Rewrite regels",
     "form.feed.label.blocklist_rules": "Blokkeer regels",

diff --git a/locale/translations/pl_PL.json b/locale/translations/pl_PL.json
@@ -268,6 +268,7 @@
     "form.feed.label.feed_username": "Subskrypcję nazwa użytkownika",
     "form.feed.label.feed_password": "Subskrypcję Hasło",
     "form.feed.label.user_agent": "Zastąp domyślny agent użytkownika",
+    "form.feed.label.cookie": "Ustawianie ciasteczek",
     "form.feed.label.scraper_rules": "Zasady ekstrakcji",
     "form.feed.label.rewrite_rules": "Reguły zapisu",
     "form.feed.label.blocklist_rules": "Zasady blokowania",

diff --git a/locale/translations/pt_BR.json b/locale/translations/pt_BR.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Nome de usuário da fonte",
     "form.feed.label.feed_password": "Senha da fonte",
     "form.feed.label.user_agent": "Sobrescrever o agente de usuário (user-agent) padrão",
+    "form.feed.label.cookie": "Definir Cookies",
     "form.feed.label.scraper_rules": "Regras do scraper",
     "form.feed.label.rewrite_rules": "Regras para o Rewrite",
     "form.feed.label.blocklist_rules": "Regras de bloqueio",

diff --git a/locale/translations/ru_RU.json b/locale/translations/ru_RU.json
@@ -268,6 +268,7 @@
     "form.feed.label.feed_username": "Имя пользователя подписки",
     "form.feed.label.feed_password": "Пароль подписки",
     "form.feed.label.user_agent": "Переопределить User Agent по умолчанию",
+    "form.feed.label.cookie": "Установить Ку́ки",
     "form.feed.label.scraper_rules": "Правила Scraper",
     "form.feed.label.rewrite_rules": "Правила Rewrite",
     "form.feed.label.blocklist_rules": "Правила блокировки",

diff --git a/locale/translations/tr_TR.json b/locale/translations/tr_TR.json
@@ -266,6 +266,7 @@
     "form.feed.label.feed_username": "Besleme Kullanıcı Adı",
     "form.feed.label.feed_password": "Besleme Parolası",
     "form.feed.label.user_agent": "Varsayılan User Agent'i Geçersiz Kıl",
+    "form.feed.label.cookie": "Çerezleri Ayarla",
     "form.feed.label.scraper_rules": "Scrapper Kuralları",
     "form.feed.label.rewrite_rules": "Yeniden Yazma Kuralları",
     "form.feed.label.blocklist_rules": "Engelleme Kuralları",

diff --git a/locale/translations/zh_CN.json b/locale/translations/zh_CN.json
@@ -264,6 +264,7 @@
     "form.feed.label.feed_username": "源用户名",
     "form.feed.label.feed_password": "源密码",
     "form.feed.label.user_agent": "覆盖默认 User-Agent",
+    "form.feed.label.cookie": "设置 Cookies",
     "form.feed.label.scraper_rules": "Scraper 规则",
     "form.feed.label.rewrite_rules": "重写规则",
     "form.feed.label.blocklist_rules": "封锁规则",

diff --git a/model/feed.go b/model/feed.go
@@ -41,6 +41,7 @@ type Feed struct {
 	BlocklistRules              string    `json:"blocklist_rules"`
 	KeeplistRules               string    `json:"keeplist_rules"`
 	UserAgent                   string    `json:"user_agent"`
+	Cookie                      string    `json:"cookie"`
 	Username                    string    `json:"username"`
 	Password                    string    `json:"password"`
 	Disabled                    bool      `json:"disabled"`
@@ -121,6 +122,7 @@ type FeedCreationRequest struct {
 	FeedURL                     string `json:"feed_url"`
 	CategoryID                  int64  `json:"category_id"`
 	UserAgent                   string `json:"user_agent"`
+	Cookie                      string `json:"cookie"`
 	Username                    string `json:"username"`
 	Password                    string `json:"password"`
 	Crawler                     bool   `json:"crawler"`
@@ -145,6 +147,7 @@ type FeedModificationRequest struct {
 	KeeplistRules               *string `json:"keeplist_rules"`
 	Crawler                     *bool   `json:"crawler"`
 	UserAgent                   *string `json:"user_agent"`
+	Cookie                      *string `json:"cookie"`
 	Username                    *string `json:"username"`
 	Password                    *string `json:"password"`
 	CategoryID                  *int64  `json:"category_id"`
@@ -192,6 +195,10 @@ func (f *FeedModificationRequest) Patch(feed *Feed) {
 		feed.UserAgent = *f.UserAgent
 	}
 
+	if f.Cookie != nil {
+		feed.Cookie = *f.Cookie
+	}
+
 	if f.Username != nil {
 		feed.Username = *f.Username
 	}

diff --git a/model/subscription.go b/model/subscription.go
@@ -8,6 +8,7 @@ package model // import "miniflux.app/model"
 type SubscriptionDiscoveryRequest struct {
 	URL                         string `json:"url"`
 	UserAgent                   string `json:"user_agent"`
+	Cookie                      string `json:"cookie"`
 	Username                    string `json:"username"`
 	Password                    string `json:"password"`
 	FetchViaProxy               bool   `json:"fetch_via_proxy"`

diff --git a/reader/handler/handler.go b/reader/handler/handler.go
@@ -39,6 +39,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
 	request := client.NewClientWithConfig(feedCreationRequest.FeedURL, config.Opts)
 	request.WithCredentials(feedCreationRequest.Username, feedCreationRequest.Password)
 	request.WithUserAgent(feedCreationRequest.UserAgent)
+	request.WithCookie(feedCreationRequest.Cookie)
 	request.AllowSelfSignedCertificates = feedCreationRequest.AllowSelfSignedCertificates
 
 	if feedCreationRequest.FetchViaProxy {
@@ -61,6 +62,7 @@ func CreateFeed(store *storage.Storage, userID int64, feedCreationRequest *model
 
 	subscription.UserID = userID
 	subscription.UserAgent = feedCreationRequest.UserAgent
+	subscription.Cookie = feedCreationRequest.Cookie
 	subscription.Username = feedCreationRequest.Username
 	subscription.Password = feedCreationRequest.Password
 	subscription.Crawler = feedCreationRequest.Crawler
@@ -124,6 +126,7 @@ func RefreshFeed(store *storage.Storage, userID, feedID int64) error {
 	request := client.NewClientWithConfig(originalFeed.FeedURL, config.Opts)
 	request.WithCredentials(originalFeed.Username, originalFeed.Password)
 	request.WithUserAgent(originalFeed.UserAgent)
+	request.WithCookie(originalFeed.Cookie)
 	request.AllowSelfSignedCertificates = originalFeed.AllowSelfSignedCertificates
 
 	if !originalFeed.IgnoreHTTPCache {

diff --git a/reader/processor/processor.go b/reader/processor/processor.go
@@ -54,6 +54,7 @@ func ProcessFeedEntries(store *storage.Storage, feed *model.Feed) {
 				entry.URL,
 				feed.ScraperRules,
 				feed.UserAgent,
+				feed.Cookie,
 				feed.AllowSelfSignedCertificates,
 			)
 
@@ -115,6 +116,7 @@ func ProcessEntryWebPage(feed *model.Feed, entry *model.Entry) error {
 		entry.URL,
 		entry.Feed.ScraperRules,
 		entry.Feed.UserAgent,
+		entry.Feed.Cookie,
 		feed.AllowSelfSignedCertificates,
 	)
 

diff --git a/reader/scraper/scraper.go b/reader/scraper/scraper.go
@@ -20,9 +20,10 @@ import (
 )
 
 // Fetch downloads a web page and returns relevant contents.
-func Fetch(websiteURL, rules, userAgent string, allowSelfSignedCertificates bool) (string, error) {
+func Fetch(websiteURL, rules, userAgent string, cookie string, allowSelfSignedCertificates bool) (string, error) {
 	clt := client.NewClientWithConfig(websiteURL, config.Opts)
 	clt.WithUserAgent(userAgent)
+	clt.WithCookie(cookie)
 	clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
 
 	response, err := clt.Get()

diff --git a/reader/subscription/finder.go b/reader/subscription/finder.go
@@ -27,13 +27,14 @@ var (
 )
 
 // FindSubscriptions downloads and try to find one or more subscriptions from an URL.
-func FindSubscriptions(websiteURL, userAgent, username, password string, fetchViaProxy, allowSelfSignedCertificates bool) (Subscriptions, *errors.LocalizedError) {
+func FindSubscriptions(websiteURL, userAgent, cookie, username, password string, fetchViaProxy, allowSelfSignedCertificates bool) (Subscriptions, *errors.LocalizedError) {
 	websiteURL = findYoutubeChannelFeed(websiteURL)
 	websiteURL = parseYoutubeVideoPage(websiteURL)
 
 	clt := client.NewClientWithConfig(websiteURL, config.Opts)
 	clt.WithCredentials(username, password)
 	clt.WithUserAgent(userAgent)
+	clt.WithCookie(cookie)
 	clt.AllowSelfSignedCertificates = allowSelfSignedCertificates
 
 	if fetchViaProxy {
@@ -62,7 +63,7 @@ func FindSubscriptions(websiteURL, userAgent, username, password string, fetchVi
 		return subscriptions, err
 	}
 
-	return tryWellKnownUrls(websiteURL, userAgent, username, password)
+	return tryWellKnownUrls(websiteURL, userAgent, cookie, username, password)
 }
 
 func parseWebPage(websiteURL string, data io.Reader) (Subscriptions, *errors.LocalizedError) {
@@ -138,7 +139,7 @@ func parseYoutubeVideoPage(websiteURL string) string {
 	return websiteURL
 }
 
-func tryWellKnownUrls(websiteURL, userAgent, username, password string) (Subscriptions, *errors.LocalizedError) {
+func tryWellKnownUrls(websiteURL, userAgent, cookie, username, password string) (Subscriptions, *errors.LocalizedError) {
 	var subscriptions Subscriptions
 	knownURLs := map[string]string{
 		"/atom.xml": "atom",
@@ -161,6 +162,7 @@ func tryWellKnownUrls(websiteURL, userAgent, username, password string) (Subscri
 		clt := client.NewClientWithConfig(fullURL, config.Opts)
 		clt.WithCredentials(username, password)
 		clt.WithUserAgent(userAgent)
+		clt.WithCookie(cookie)
 
 		// Some websites redirects unknown URLs to the home page.
 		// As result, the list of known URLs is returned to the subscription list.

diff --git a/storage/entry_query_builder.go b/storage/entry_query_builder.go
@@ -242,6 +242,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
 			f.rewrite_rules,
 			f.crawler,
 			f.user_agent,
+			f.cookie,
 			fi.icon_id,
 			u.timezone
 		FROM
@@ -303,6 +304,7 @@ func (e *EntryQueryBuilder) GetEntries() (model.Entries, error) {
 			&entry.Feed.RewriteRules,
 			&entry.Feed.Crawler,
 			&entry.Feed.UserAgent,
+			&entry.Feed.Cookie,
 			&iconID,
 			&tz,
 		)

diff --git a/storage/feed.go b/storage/feed.go
@@ -195,6 +195,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 			last_modified_header,
 			crawler,
 			user_agent,
+			cookie,
 			username,
 			password,
 			disabled,
@@ -207,7 +208,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 			fetch_via_proxy
 		)
 		VALUES
-			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19)
+			($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20)
 		RETURNING
 			id
 	`
@@ -222,6 +223,7 @@ func (s *Storage) CreateFeed(feed *model.Feed) error {
 		feed.LastModifiedHeader,
 		feed.Crawler,
 		feed.UserAgent,
+		feed.Cookie,
 		feed.Username,
 		feed.Password,
 		feed.Disabled,
@@ -282,15 +284,16 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
 			keeplist_rules=$13,
 			crawler=$14,
 			user_agent=$15,
-			username=$16,
-			password=$17,
-			disabled=$18,
-			next_check_at=$19,
-			ignore_http_cache=$20,
-			allow_self_signed_certificates=$21,
-			fetch_via_proxy=$22
+			cookie=$16,
+			username=$17,
+			password=$18,
+			disabled=$19,
+			next_check_at=$20,
+			ignore_http_cache=$21,
+			allow_self_signed_certificates=$22,
+			fetch_via_proxy=$23
 		WHERE
-			id=$23 AND user_id=$24
+			id=$24 AND user_id=$25
 	`
 	_, err = s.db.Exec(query,
 		feed.FeedURL,
@@ -308,6 +311,7 @@ func (s *Storage) UpdateFeed(feed *model.Feed) (err error) {
 		feed.KeeplistRules,
 		feed.Crawler,
 		feed.UserAgent,
+		feed.Cookie,
 		feed.Username,
 		feed.Password,
 		feed.Disabled,

diff --git a/storage/feed_query_builder.go b/storage/feed_query_builder.go
@@ -159,6 +159,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
 			f.keeplist_rules,
 			f.crawler,
 			f.user_agent,
+			f.cookie,
 			f.username,
 			f.password,
 			f.ignore_http_cache,
@@ -218,6 +219,7 @@ func (f *FeedQueryBuilder) GetFeeds() (model.Feeds, error) {
 			&feed.KeeplistRules,
 			&feed.Crawler,
 			&feed.UserAgent,
+			&feed.Cookie,
 			&feed.Username,
 			&feed.Password,
 			&feed.IgnoreHTTPCache,