Skip to content

Commit

Permalink
Implemented the new polling sources to the new interface
Browse files Browse the repository at this point in the history
  • Loading branch information
Emyrk committed Aug 6, 2019
1 parent 1d9210d commit f7bbbd4
Show file tree
Hide file tree
Showing 14 changed files with 304 additions and 162 deletions.
1 change: 1 addition & 0 deletions defaultconfig.ini
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@

# This section must ONLY include data sources and their priorities. Any configuration
# related to a source should be specified in the [Oracle] section.
# -1 == disabled
[OracleDataSources]
APILayer=1
ExchangeRatesAPI=0
Expand Down
17 changes: 4 additions & 13 deletions polling/apilayer.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,10 @@ type APILayerDataSource struct {
lastPeg PegAssets
}

func NewAPILayerDataSource(config *config.Config) *APILayerDataSource {
func NewAPILayerDataSource(config *config.Config) (*APILayerDataSource, error) {
s := new(APILayerDataSource)
s.config = config
return s
return s, nil
}

func (d *APILayerDataSource) Name() string {
Expand All @@ -52,24 +52,15 @@ func (d *APILayerDataSource) FetchPegPrices() (peg PegAssets, err error) {
for _, currencyISO := range d.SupportedPegs() {
// Search for USDXXX pairs
if v, ok := resp.Quotes["USD"+currencyISO]; ok {
peg[currencyISO] = PegItem{Value: v, When: timestamp}
peg[currencyISO] = PegItem{Value: v, When: timestamp, WhenUnix: timestamp.Unix()}
}
}

return
}

func (d *APILayerDataSource) FetchPegPrice(peg string) (i PegItem, err error) {
p, err := d.FetchPegPrices()
if err != nil {
return
}

item, ok := p[peg]
if !ok {
return i, fmt.Errorf("peg not found")
}
return item, nil
return FetchPegPrice(peg, d.FetchPegPrices)
}

// ----
Expand Down
25 changes: 1 addition & 24 deletions polling/apilayer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,35 +3,12 @@
package polling_test

import (
"net/http"
"testing"

"github.com/pegnet/pegnet/common"
. "github.com/pegnet/pegnet/polling"
"github.com/pegnet/pegnet/testutils"
"github.com/zpatrick/go-config"
)

// TestFixedApiLayerPeggedAssets tests all the crypto assets are found on ApiLayer
func TestFixedApiLayerPeggedAssets(t *testing.T) {
defer func() { http.DefaultClient = &http.Client{} }() // Don't leave http broken
c := config.NewConfig([]config.Provider{common.NewUnitTestConfigProvider()})

// Set default http client to return what we expect from apilayer
cl := testutils.GetClientWithFixedResp([]byte(apiLayerReponse))
http.DefaultClient = cl

peg := make(PegAssets)
err := APILayerInterface(c, peg)
if err != nil {
t.Error(err)
}
for _, asset := range common.CurrencyAssets {
_, ok := peg[asset]
if !ok {
t.Errorf("Missing %s", asset)
}
}
FixedDataSourceTest(t, "APILayer", []byte(apiLayerReponse))
}

var apiLayerReponse = `{"success":true,"terms":"https:\/\/currencylayer.com\/terms","privacy":"https:\/\/currencylayer.com\/privacy","timestamp":1563828666,"source":"USD","quotes":{"USDAED":3.67295,"USDAFN":79.850338,"USDALL":108.870258,"USDAMD":476.180354,"USDANG":1.78505,"USDAOA":346.555499,"USDARS":42.456988,"USDAUD":1.421297,"USDAWG":1.79975,"USDAZN":1.705017,"USDBAM":1.742903,"USDBBD":2.01975,"USDBDT":84.51296,"USDBGN":1.745099,"USDBHD":0.377005,"USDBIF":1851,"USDBMD":1,"USDBND":1.35065,"USDBOB":6.91285,"USDBRL":3.739797,"USDBSD":1.00025,"USDBTC":9.7194188e-5,"USDBTN":68.914102,"USDBWP":10.577987,"USDBYN":2.017397,"USDBYR":19600,"USDBZD":2.01575,"USDCAD":1.31165,"USDCDF":1665.499211,"USDCHF":0.9821,"USDCLF":0.02497,"USDCLP":689.000158,"USDCNY":6.881102,"USDCOP":3174.15,"USDCRC":574.735014,"USDCUC":1,"USDCUP":26.5,"USDCVE":98.222497,"USDCZK":22.786402,"USDDJF":177.719751,"USDDKK":6.66095,"USDDOP":51.044963,"USDDZD":119.260028,"USDEGP":16.619889,"USDERN":14.999739,"USDETB":29.004983,"USDEUR":0.892204,"USDFJD":2.1268,"USDFKP":0.80079,"USDGBP":0.8015,"USDGEL":2.875042,"USDGGP":0.801578,"USDGHS":5.389403,"USDGIP":0.80079,"USDGMD":49.964996,"USDGNF":9237.500707,"USDGTQ":7.665351,"USDGYD":208.824973,"USDHKD":7.81065,"USDHNL":24.650026,"USDHRK":6.591016,"USDHTG":94.061501,"USDHUF":290.269929,"USDIDR":13939,"USDILS":3.527506,"USDIMP":0.801578,"USDINR":68.929733,"USDIQD":1190,"USDIRR":42105.000124,"USDISK":124.819818,"USDJEP":0.801578,"USDJMD":134.619994,"USDJOD":0.707697,"USDJPY":107.87297,"USDKES":103.59735,"USDKGS":69.648033,"USDKHR":4082.999949,"USDKMF":438.800805,"USDKPW":900.064657,"USDKRW":1176.860062,"USDKWD":0.30425,"USDKYD":0.833405,"USDKZT":385.740062,"USDLAK":8684.999964,"USDLBP":1511.650177,"USDLKR":175.870017,"USDLRD":201.624975,"USDLSL":13.839987,"USDLTL":2.95274,"USDLVL":0.60489,"USDLYD":1.404995,"USDMAD":9.611502,"USDMDL":17.525499,"USDMGA":3607.499758,"USDMKD":54.663501,"USDMMK":1518.100677,"USDMNT":2664.879598,"USDMOP":8.04445,"USDMRO":357.000346,"USDMUR":35.898501,"USDMVR":15.449644,"USDMWK":760.054989,"USDMXN":19.055995,"USDMYR":4.112402,"USDMZN":61.814976,"USDNAD":13.839653,"USDNGN":360.000055,"USDNIO":33.50191,"USDNOK":8.61012,"USDNPR":110.244994,"USDNZD":1.478875,"USDOMR":0.38499,"USDPAB":1.00025,"USDPEN":3.285497,"USDPGK":3.389744,"USDPHP":51.109569,"USDPKR":159.669867,"USDPLN":3.78865,"USDPYG":5977.550021,"USDQAR":3.64175,"USDRON":4.212597,"USDRSD":105.029919,"USDRUB":63.102504,"USDRWF":910,"USDSAR":3.750502,"USDSBD":8.27135,"USDSCR":13.742494,"USDSDG":45.114502,"USDSEK":9.412501,"USDSGD":1.3609,"USDSHP":1.320899,"USDSLL":9250.000056,"USDSOS":580.000449,"USDSRD":7.458009,"USDSTD":21560.79,"USDSVC":8.75025,"USDSYP":515.000005,"USDSZL":13.839832,"USDTHB":30.849747,"USDTJS":9.430199,"USDTMT":3.5,"USDTND":2.86375,"USDTOP":2.267899,"USDTRY":5.679735,"USDTTD":6.77465,"USDTWD":31.072029,"USDTZS":2299.198699,"USDUAH":25.783993,"USDUGX":3695.203866,"USDUSD":1,"USDUYU":35.129707,"USDUZS":8630.000157,"USDVEF":9.987502,"USDVND":23227.5,"USDVUV":114.779918,"USDWST":2.602798,"USDXAF":584.559787,"USDXAG":0.061126,"USDXAU":0.000702,"USDXCD":2.70255,"USDXDR":0.7239,"USDXOF":591.501104,"USDXPF":106.697048,"USDYER":250.297342,"USDZAR":13.868106,"USDZMK":9001.20624,"USDZMW":12.825506,"USDZWL":322.000001}}`
25 changes: 20 additions & 5 deletions polling/assets.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"fmt"
"regexp"
"sort"
"strings"

"github.com/pegnet/pegnet/common"
"github.com/zpatrick/go-config"
Expand All @@ -12,9 +13,15 @@ import (
func NewDataSource(source string, config *config.Config) (IDataSource, error) {
switch source {
case "APILayer":
return NewAPILayerDataSource(config), nil
return NewAPILayerDataSource(config)
case "CoinCap":
return NewCoinCapDataSource(config), nil
return NewCoinCapDataSource(config)
case "ExchangeRates":
return NewExchangeRatesDataSource(config)
case "Kitco":
return NewKitcoDataSource(config)
case "OpenExchangeRates":
return NewOpenExchangeRatesDataSource(config)
}
return nil, fmt.Errorf("%s is not a supported data source", source)
}
Expand Down Expand Up @@ -59,14 +66,22 @@ func NewDataSources(config *config.Config) *DataSources {

for setting, _ := range allSettings {
if datasourceRegex.Match([]byte(setting)) {
s, err := NewDataSource(setting, config)
common.CheckAndPanic(err)

// Get the priority. Priorities can be the same, then we'll sort
// alphabetically to keep the results deterministic
p, err := config.Int(setting)
common.CheckAndPanic(err)

if p == -1 {
continue // This source is disabled
}

source := strings.Split(setting, ".")
if len(source) != 2 {
panic(common.DetailError(fmt.Errorf("expect only 1 '.' in a setting. Found %s", setting)))
}
s, err := NewDataSource(source[1], config)
common.CheckAndPanic(err)

// Add to our lists
d.PriorityList = append(d.PriorityList, DataSourceWithPriority{DataSource: s, Priority: p})
d.DataSources[s.Name()] = s
Expand Down
15 changes: 3 additions & 12 deletions polling/coincap.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ type CoinCapDataSource struct {
lastPeg PegAssets
}

func NewCoinCapDataSource(config *config.Config) *CoinCapDataSource {
func NewCoinCapDataSource(config *config.Config) (*CoinCapDataSource, error) {
s := new(CoinCapDataSource)
s.config = config
return s
return s, nil
}

func (d *CoinCapDataSource) Name() string {
Expand Down Expand Up @@ -89,16 +89,7 @@ func (d *CoinCapDataSource) FetchPegPrices() (peg PegAssets, err error) {
}

func (d *CoinCapDataSource) FetchPegPrice(peg string) (i PegItem, err error) {
p, err := d.FetchPegPrices()
if err != nil {
return
}

item, ok := p[peg]
if !ok {
return i, fmt.Errorf("peg not found")
}
return item, nil
return FetchPegPrice(peg, d.FetchPegPrices)
}

// -----
Expand Down
36 changes: 2 additions & 34 deletions polling/coincap_test.go

Large diffs are not rendered by default.

17 changes: 17 additions & 0 deletions polling/datasource.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package polling

import "fmt"

// IDataSource is the implementation all data sources need to adheer to.
type IDataSource interface {
// Include some human friendly things.
Expand All @@ -18,3 +20,18 @@ type IDataSource interface {
// be defined up front.
SupportedPegs() []string
}

// FetchPegPrice is because this implementation is the same for each exchange and GoLang's
// inheritance makes child structs referencing parent structs weird.
func FetchPegPrice(peg string, FetchPegPrices func() (peg PegAssets, err error)) (i PegItem, err error) {
p, err := FetchPegPrices()
if err != nil {
return
}

item, ok := p[peg]
if !ok {
return i, fmt.Errorf("peg not found")
}
return item, nil
}
64 changes: 64 additions & 0 deletions polling/datasource_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package polling_test

import (
"net/http"
"testing"

"github.com/pegnet/pegnet/common"
"github.com/pegnet/pegnet/polling"
"github.com/pegnet/pegnet/testutils"
"github.com/zpatrick/go-config"
)

// FixedDataSourceTest will test the parsing of the data source using the fixed response
func FixedDataSourceTest(t *testing.T, source string, fixed []byte) {
defer func() { http.DefaultClient = &http.Client{} }() // Don't leave http broken

c := config.NewConfig([]config.Provider{common.NewUnitTestConfigProvider()})

// Set default http client to return what we expect from apilayer
cl := testutils.GetClientWithFixedResp(fixed)
http.DefaultClient = cl

s, err := polling.NewDataSource(source, c)
if err != nil {
t.Error(err)
}

pegs, err := s.FetchPegPrices()
if err != nil {
t.Error(err)
}

for _, asset := range s.SupportedPegs() {
_, ok := pegs[asset]
if !ok {
t.Errorf("Missing %s", asset)
}
}
}

// ActualDataSourceTest actually fetches the resp over the internet
func ActualDataSourceTest(t *testing.T, source string) {
defer func() { http.DefaultClient = &http.Client{} }() // Don't leave http broken

c := config.NewConfig([]config.Provider{common.NewUnitTestConfigProvider()})
http.DefaultClient = &http.Client{}

s, err := polling.NewDataSource(source, c)
if err != nil {
t.Error(err)
}

pegs, err := s.FetchPegPrices()
if err != nil {
t.Error(err)
}

for _, asset := range s.SupportedPegs() {
_, ok := pegs[asset]
if !ok {
t.Errorf("Missing %s", asset)
}
}
}
55 changes: 55 additions & 0 deletions polling/exchangeratesapi.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,67 @@ import (
"fmt"
"io/ioutil"
"net/http"
"time"

"github.com/pegnet/pegnet/common"

"github.com/cenkalti/backoff"
log "github.com/sirupsen/logrus"
"github.com/zpatrick/go-config"
)

// ExchangeRatesDataSource is the datasource at "https://exchangeratesapi.io"
type ExchangeRatesDataSource struct {
config *config.Config
lastPeg PegAssets
}

func NewExchangeRatesDataSource(config *config.Config) (*ExchangeRatesDataSource, error) {
s := new(ExchangeRatesDataSource)
s.config = config
return s, nil
}

func (d *ExchangeRatesDataSource) Name() string {
return "ExchangeRates"
}

func (d *ExchangeRatesDataSource) Url() string {
return "https://exchangeratesapi.io"
}

func (d *ExchangeRatesDataSource) SupportedPegs() []string {
return common.CurrencyAssets
}

func (d *ExchangeRatesDataSource) FetchPegPrices() (peg PegAssets, err error) {
resp, err := CallExchangeRatesAPI(d.config)
if err != nil {
return nil, err
}

peg = make(map[string]PegItem)

timestamp, err := time.Parse("2006-01-02", resp.Date)
if err != nil {
return nil, err
}

for _, currencyISO := range d.SupportedPegs() {
if v, ok := resp.Rates[currencyISO]; ok {
peg[currencyISO] = PegItem{Value: v, When: timestamp, WhenUnix: timestamp.Unix()}
}
}

return
}

func (d *ExchangeRatesDataSource) FetchPegPrice(peg string) (i PegItem, err error) {
return FetchPegPrice(peg, d.FetchPegPrices)
}

// ------

type ExchangeRatesAPIResponse struct {
Date string `json:"date"`
Base string `json:"base"`
Expand Down
37 changes: 2 additions & 35 deletions polling/exchangeratesapi_test.go
Original file line number Diff line number Diff line change
@@ -1,49 +1,16 @@
package polling_test

import (
"net/http"
"testing"

"github.com/pegnet/pegnet/common"
. "github.com/pegnet/pegnet/polling"
"github.com/pegnet/pegnet/testutils"
"github.com/zpatrick/go-config"
)

// TestActualExchangeRatesPeggedAssets tests all the crypto assets are found on exchangerates over the net
func TestActualExchangeRatesPeggedAssets(t *testing.T) {
c := config.NewConfig([]config.Provider{common.NewUnitTestConfigProvider()})
peg := make(PegAssets)

http.DefaultClient = &http.Client{}

ExchangeRatesAPIInterface(c, peg)
for _, asset := range common.CurrencyAssets {
_, ok := peg[asset]
if !ok {
t.Errorf("Missing %s", asset)
}
}
ActualDataSourceTest(t, "ExchangeRates")
}

// TestFixedExchangeRatesPeggedAssets tests all the crypto assets are found on exchangerates from fixed
func TestFixedExchangeRatesPeggedAssets(t *testing.T) {
defer func() { http.DefaultClient = &http.Client{} }() // Don't leave http broken

c := config.NewConfig([]config.Provider{common.NewUnitTestConfigProvider()})

// Set default http client to return what we expect from apilayer
cl := testutils.GetClientWithFixedResp([]byte(exchangeRateResponse))
http.DefaultClient = cl

peg := make(PegAssets)
ExchangeRatesAPIInterface(c, peg)
for _, asset := range common.CurrencyAssets {
_, ok := peg[asset]
if !ok {
t.Errorf("Missing %s", asset)
}
}
FixedDataSourceTest(t, "ExchangeRates", []byte(exchangeRateResponse))
}

var exchangeRateResponse = `{"rates":{"CAD":1.3076237182,"HKD":7.8096299599,"ISK":124.7436469015,"PHP":51.1110120374,"DKK":6.6571555952,"HUF":289.7458760588,"CZK":22.7677218012,"GBP":0.8022113241,"RON":4.210878288,"SEK":9.4016941596,"IDR":13945.0022291574,"INR":68.9331252786,"BRL":3.7434685689,"RUB":62.9982166741,"HRK":6.5871600535,"JPY":107.9179670085,"THB":30.8452964779,"CHF":0.9814534106,"EUR":0.8916629514,"MYR":4.1129736959,"BGN":1.7439144004,"TRY":5.6818546589,"CNY":6.8807846634,"NOK":8.5844850646,"NZD":1.4750780205,"ZAR":13.8922871155,"USD":1.0,"MXN":19.0295140437,"SGD":1.3606776638,"AUD":1.4188140883,"ILS":3.5329469461,"KRW":1177.2982612572,"PLN":3.7877842176},"base":"USD","date":"2019-07-22"}`
Loading

0 comments on commit f7bbbd4

Please sign in to comment.