Skip to content

Commit

Permalink
change crawler
Browse files Browse the repository at this point in the history
  • Loading branch information
Sagleft committed Nov 23, 2021
1 parent 692872c commit 22fb543
Show file tree
Hide file tree
Showing 2 changed files with 8 additions and 19 deletions.
25 changes: 8 additions & 17 deletions client.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package goinvest

import (
"encoding/json"
"log"
"fmt"

"github.com/foolin/pagser"
"github.com/PuerkitoBio/goquery"
)

// NewClient - create new Client
Expand All @@ -15,26 +15,17 @@ func NewClient() *Client {
// GetScreener data
func (c *Client) GetScreener(task ScreenerTask) (*ScreenerResponse, error) {
url := c.buildScreenerRequestURL(task)
pageBytes, err := c.get(url)
if err != nil {
return nil, err
}

// new default config
p := pagser.New()

// data parser model
var data screenerPageData

// parse html data
err = p.Parse(&data, string(pageBytes))
// check error
// Load the HTML document
doc, err := goquery.NewDocument(url)
if err != nil {
return nil, err
}

// print data
log.Printf("Page data json: \n-------------\n%v\n-------------\n", toJSON(data))
doc.Find("table#resultsTable > tbody > tr").Each(func(i int, s *goquery.Selection) {
title := s.Find("td.symbol").Text()
fmt.Println(title)
})

return nil, nil
}
Expand Down
2 changes: 0 additions & 2 deletions structs.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,5 +8,3 @@ type ScreenerTask struct{}

// ScreenerResponse data
type ScreenerResponse struct{}

type screenerPageData struct{}

0 comments on commit 22fb543

Please sign in to comment.