refactor auth api

api.go

@@ -14,51 +14,79 @@ const bearerToken string = "AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K
 func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
 	s.wg.Wait()
 	if s.delay > 0 {
-		defer func() {
-			s.wg.Add(1)
-			go func() {
-				time.Sleep(time.Second * time.Duration(s.delay))
-				s.wg.Done()
-			}()
-		}()
+		defer s.delayRequest()
 	}
 
+	if err := s.prepareRequest(req); err != nil {
+		return err
+	}
+
+	resp, err := s.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	return s.handleResponse(resp, target)
+}
+
+func (s *Scraper) delayRequest() {
+	s.wg.Add(1)
+	go func() {
+		time.Sleep(time.Second * time.Duration(s.delay))
+		s.wg.Done()
+	}()
+}
+
+func (s *Scraper) prepareRequest(req *http.Request) error {
+	req.Header.Set("User-Agent", s.userAgent)
+
 	if !s.isLogged {
-		if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) {
-			err := s.GetGuestToken()
-			if err != nil {
-				return err
-			}
+		if err := s.setGuestToken(req); err != nil {
+			return err
 		}
-		req.Header.Set("X-Guest-Token", s.guestToken)
 	}
 
+	s.setAuthorizationHeader(req)
+	s.setCSRFToken(req)
+
+	return nil
+}
+
+func (s *Scraper) setGuestToken(req *http.Request) error {
+	if !s.IsGuestToken() || s.guestCreatedAt.Before(time.Now().Add(-time.Hour*3)) {
+		if err := s.GetGuestToken(); err != nil {
+			return err
+		}
+	}
+	req.Header.Set("X-Guest-Token", s.guestToken)
+	return nil
+}
+
+func (s *Scraper) setAuthorizationHeader(req *http.Request) {
 	if s.oAuthToken != "" && s.oAuthSecret != "" {
 		req.Header.Set("Authorization", s.sign(req.Method, req.URL))
 	} else {
 		req.Header.Set("Authorization", "Bearer "+s.bearerToken)
 	}
+}
 
+func (s *Scraper) setCSRFToken(req *http.Request) {
 	for _, cookie := range s.client.Jar.Cookies(req.URL) {
 		if cookie.Name == "ct0" {
 			req.Header.Set("X-CSRF-Token", cookie.Value)
 			break
 		}
 	}
+}
 
-	resp, err := s.client.Do(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
+func (s *Scraper) handleResponse(resp *http.Response, target interface{}) error {
 	content, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return err
 	}
 
-	statusOK := resp.StatusCode >= 200 && resp.StatusCode < 300
-	if !statusOK {
+	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("response status %s: %s", resp.Status, content)
 	}
 
@@ -69,6 +97,7 @@ func (s *Scraper) RequestAPI(req *http.Request, target interface{}) error {
 	if target == nil {
 		return nil
 	}
+
 	return json.Unmarshal(content, target)
 }
 

auth.go

@@ -84,7 +84,7 @@ func (s *Scraper) getFlow(data map[string]interface{}) (*flow, error) {
 	headers := http.Header{
 		"Authorization":             []string{"Bearer " + s.bearerToken},
 		"Content-Type":              []string{"application/json"},
-		"User-Agent":                []string{"TwitterAndroid/99"},
+		"User-Agent":                []string{s.userAgent},
 		"X-Guest-Token":             []string{s.guestToken},
 		"X-Twitter-Auth-Type":       []string{"OAuth2Client"},
 		"X-Twitter-Active-User":     []string{"yes"},

scraper.go

@@ -27,6 +27,7 @@ type Scraper struct {
 	oAuthToken     string
 	oAuthSecret    string
 	proxy          string
+	userAgent      string
 	searchMode     SearchMode
 	wg             sync.WaitGroup
 }
@@ -49,12 +50,14 @@ const (
 
 // default http client timeout
 const DefaultClientTimeout = 10 * time.Second
+const DefaultUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36"
 
 // New creates a Scraper object
 func New() *Scraper {
 	jar, _ := cookiejar.New(nil)
 	return &Scraper{
 		bearerToken: bearerToken,
+		userAgent:   DefaultUserAgent,
 		client: &http.Client{
 			Jar:     jar,
 			Timeout: DefaultClientTimeout,
@@ -169,3 +172,11 @@ func (s *Scraper) SetProxy(proxyAddr string) error {
 	}
 	return errors.New("only support http(s) or socks5 protocol")
 }
+
+func (s *Scraper) SetUserAgent(userAgent string) {
+	s.userAgent = userAgent
+}
+
+func (s *Scraper) GetUserAgent() string {
+	return s.userAgent
+}