Skip to content

Commit

Permalink
Fetch token prices from data-api (#1019)
Browse files Browse the repository at this point in the history
* fetch prices from data-api

* test data-api service

* update data-api url

* include self url

* include features

* fix configs

* add data api feature config

* SERVICES-1446 fix some tests

* SERVICES-1446 fix some tests  (2)

* SERVICES-1446 skip tests until ES are solved

* SERVICES-1446 check getEsdtTokenPrice when API feature isDataApiFeatureEnabled is false

* fix errors

* mock error logger

* add fields interceptors to data api calls

* SERVICES-1446 replace in nft contoller tests beforeEach with beforeAll

* elrond => multiversx

* tokens as object

* SERVICES-1446 update data-api tests

---------

Co-authored-by: catalinfaurpaul <[email protected]>
Co-authored-by: tanghel <[email protected]>
  • Loading branch information
3 people authored Mar 28, 2023
1 parent 4a148e8 commit eed4751
Show file tree
Hide file tree
Showing 25 changed files with 419 additions and 90 deletions.
5 changes: 4 additions & 1 deletion config/config.devnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ features:
exchange:
enabled: false
serviceUrl: 'https://devnet-graph.xexchange.com/graphql'
dataApi:
enabled: false
serviceUrl: 'https://devnet-data-api.multiversx.com'
auth:
enabled: false
maxExpirySeconds: 86400
Expand All @@ -60,7 +63,7 @@ aws:
s3Bucket: 'devnet-media.elrond.com'
s3Region: ''
urls:
self: 'https://devnet-api.elrond.com'
self: 'https://devnet-api.multiversx.com'
api:
- 'https://devnet-api.multiversx.com'
- 'https://testnet-api.multiversx.com'
Expand Down
13 changes: 10 additions & 3 deletions config/config.e2e-mocked.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,15 @@ api:
public: true
private: true
graphql: true

features:
dataApi:
enabled: false
serviceUrl: 'https://data-api.multiversx.com'
auth:
enabled: false
maxExpirySeconds: 86400
acceptedOrigins:
- ''
cron:
transactionProcessor: false
transactionProcessorMaxLookBehind: 1000
Expand All @@ -15,14 +23,13 @@ flags:
useTracing: false
indexer-v3: true
urls:
self: 'https://api.multiversx.com'
api:
- 'https://api.multiversx.com'
- 'https://devnet-api.multiversx.com'
- 'https://testnet-api.multiversx.com'
elastic:
- 'https://index.multiversx.com'
mex:
- 'https://mex-indexer.elrond.com'
gateway:
- 'https://gateway.multiversx.com'
verifier: 'https://play-api.multiversx.com'
Expand Down
13 changes: 10 additions & 3 deletions config/config.e2e.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,15 @@ api:
public: true
private: true
graphql: true

features:
dataApi:
enabled: false
serviceUrl: 'https://data-api.multiversx.com'
auth:
enabled: false
maxExpirySeconds: 86400
acceptedOrigins:
- ''
cron:
transactionProcessor: false
transactionProcessorMaxLookBehind: 1000
Expand All @@ -15,14 +23,13 @@ flags:
useTracing: false
indexer-v3: true
urls:
self: 'https://api.multiversx.com'
api:
- 'https://api.multiversx.com'
- 'https://devnet-api.multiversx.com'
- 'https://testnet-api.multiversx.com'
elastic:
- 'https://index.multiversx.com'
mex:
- 'https://mex-indexer.elrond.com'
gateway:
- 'https://gateway.multiversx.com'
verifier: 'https://play-api.multiversx.com'
Expand Down
7 changes: 4 additions & 3 deletions config/config.mainnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ features:
exchange:
enabled: false
serviceUrl: 'https://graph.xexchange.com/graphql'
dataApi:
enabled: false
serviceUrl: 'https://data-api.multiversx.com'
auth:
enabled: false
maxExpirySeconds: 86400
Expand All @@ -60,15 +63,13 @@ aws:
s3Bucket: 'media.elrond.com'
s3Region: ''
urls:
self: 'https://api.elrond.com'
self: 'https://api.multiversx.com'
api:
- 'https://api.multiversx.com'
- 'https://devnet-api.multiversx.com'
- 'https://testnet-api.multiversx.com'
elastic:
- 'https://index.multiversx.com'
mex:
- 'https://mex-indexer.elrond.com'
gateway:
- 'https://gateway.multiversx.com'
verifier: 'https://play-api.multiversx.com'
Expand Down
7 changes: 5 additions & 2 deletions config/config.testnet.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,9 @@ features:
exchange:
enabled: false
serviceUrl: 'https://testnet-graph.xexchange.com/graphql'
dataApi:
enabled: false
serviceUrl: 'https://testnet-data-api.multiversx.com'
auth:
enabled: false
maxExpirySeconds: 86400
Expand All @@ -60,13 +63,13 @@ aws:
s3Bucket: 'testnet-media.elrond.com'
s3Region: ''
urls:
self: 'https://testnet-api.elrond.com'
self: 'https://testnet-api.multiversx.com'
api:
- 'https://testnet-api.multiversx.com'
- 'https://devnet-api.multiversx.com'
- 'https://api.multiversx.com'
elastic:
- 'https://testnet-index.elrond.com'
- 'https://testnet-index.multiversx.com'
gateway:
- 'https://testnet-gateway.multiversx.com'
verifier: 'https://play-api.multiversx.com'
Expand Down
2 changes: 1 addition & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 13 additions & 9 deletions src/common/api-config/api.config.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -77,15 +77,6 @@ export class ApiConfigService {
return elasticUrls[Math.floor(Math.random() * elasticUrls.length)];
}

getMexUrl(): string {
const mexUrls = this.configService.get<string>('urls.mex');
if (mexUrls) {
return mexUrls[Math.floor(Math.random() * mexUrls.length)];
}

return '';
}

getIpfsUrl(): string {
return this.configService.get<string>('urls.ipfs') ?? 'https://ipfs.io/ipfs';
}
Expand Down Expand Up @@ -798,4 +789,17 @@ export class ApiConfigService {
getNativeAuthMaxExpirySeconds(): number {
return this.configService.get<number>('features.auth.maxExpirySeconds') ?? Constants.oneDay();
}

isDataApiFeatureEnabled(): boolean {
return this.configService.get<boolean>('features.dataApi.enabled') ?? false;
}

getDataApiServiceUrl(): string {
const serviceUrl = this.configService.get<string>('features.dataApi.serviceUrl');
if (!serviceUrl) {
throw new Error('No data-api service url present');
}

return serviceUrl;
}
}
20 changes: 20 additions & 0 deletions src/common/data-api/data-api.module.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
import { Global, Module } from "@nestjs/common";
import { DynamicModuleUtils } from "src/utils/dynamic.module.utils";
import { ApiConfigModule } from "../api-config/api.config.module";
import { DataApiService } from "./data-api.service";

@Global()
@Module({
imports: [
ApiConfigModule,
DynamicModuleUtils.getApiModule(),
DynamicModuleUtils.getCachingModule(),
],
providers: [
DataApiService,
],
exports: [
DataApiService,
],
})
export class DataApiModule { }
89 changes: 89 additions & 0 deletions src/common/data-api/data-api.service.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import { Injectable } from "@nestjs/common";
import { ApiConfigService } from "../api-config/api.config.service";
import { ApiService, CachingService, OriginLogger } from "@multiversx/sdk-nestjs";
import { DataApiToken } from "./entities/data-api.token";
import { CacheInfo } from "src/utils/cache.info";

@Injectable()
export class DataApiService {
private readonly logger = new OriginLogger(DataApiService.name);

constructor(
private readonly apiConfigService: ApiConfigService,
private readonly apiService: ApiService,
private readonly cachingService: CachingService,
) { }

public async getEgldPrice(timestamp?: number): Promise<number | undefined> {
return await this.getEsdtTokenPrice('EGLD', timestamp);
}

public async getEsdtTokenPrice(identifier: string, timestamp?: number): Promise<number | undefined> {
return await this.cachingService.getOrSetCache(
CacheInfo.DataApiTokenPrice(identifier, timestamp).key,
async () => await this.getEsdtTokenPriceRaw(identifier, timestamp),
CacheInfo.DataApiTokenPrice(identifier, timestamp).ttl
);
}

private async getEsdtTokenPriceRaw(identifier: string, timestamp?: number): Promise<number | undefined> {
if (!this.apiConfigService.isDataApiFeatureEnabled()) {
return undefined;
}

const token = await this.getDataApiToken(identifier);
if (!token) {
return undefined;
}

try {
const priceDate = timestamp ? new Date(timestamp * 1000).toISODateString() : undefined;
const dateQuery = priceDate ? `&date=${priceDate}` : '';
const priceUrl = `${this.apiConfigService.getDataApiServiceUrl()}/v1/quotes/${token.market}/${token.identifier}?extract=price${dateQuery}`;

const response = await this.apiService.get(priceUrl);
return response?.data;
} catch (error) {
this.logger.error(`An unexpected error occurred while fetching price for token ${identifier} from Data API.`);
this.logger.error(error);
}

return undefined;
}

public async getDataApiToken(identifier: string): Promise<DataApiToken | undefined> {
const tokens = await this.getDataApiTokens();
return tokens[identifier];
}

public async getDataApiTokens(): Promise<Record<string, DataApiToken>> {
return await this.cachingService.getOrSetCache(
CacheInfo.DataApiTokens.key,
async () => await this.getDataApiTokensRaw(),
CacheInfo.DataApiTokens.ttl
);
}

public async getDataApiTokensRaw(): Promise<Record<string, DataApiToken>> {
if (!this.apiConfigService.isDataApiFeatureEnabled()) {
return {};
}

try {
const [cexTokensRaw, xExchangeTokensRaw] = await Promise.all([
this.apiService.get(`${this.apiConfigService.getDataApiServiceUrl()}/v1/tokens/cex?fields=identifier`),
this.apiService.get(`${this.apiConfigService.getDataApiServiceUrl()}/v1/tokens/xexchange?fields=identifier`),
]);

const cexTokens: DataApiToken[] = cexTokensRaw.data.map((token: any) => new DataApiToken({ identifier: token.identifier, market: 'cex' }));
const xExchangeTokens: DataApiToken[] = xExchangeTokensRaw.data.map((token: any) => new DataApiToken({ identifier: token.identifier, market: 'xexchange' }));

const tokens = [...cexTokens, ...xExchangeTokens].toRecord<DataApiToken>(x => x.identifier);
return tokens;
} catch (error) {
this.logger.error(`An unexpected error occurred while fetching tokens from Data API.`);
this.logger.error(error);
return {};
}
}
}
8 changes: 8 additions & 0 deletions src/common/data-api/entities/data-api.token.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
export class DataApiToken {
constructor(init?: Partial<DataApiToken>) {
Object.assign(this, init);
}

identifier: string = '';
market: 'cex' | 'xexchange' = 'cex';
}
6 changes: 0 additions & 6 deletions src/common/plugins/plugin.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,5 @@ export class PluginService {

async batchProcessNfts(_nfts: Nft[], _withScamInfo?: boolean): Promise<void> { }

// eslint-disable-next-line require-await
async getEgldPrice(_timestamp?: number): Promise<number | undefined> { return undefined; }

// eslint-disable-next-line require-await
async getEsdtTokenPrice(_identifier: string, _timestamp?: number): Promise<number | undefined> { return undefined; }

async processAbout(_about: About): Promise<void> { }
}
6 changes: 3 additions & 3 deletions src/crons/cache.warmer/cache.warmer.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,14 @@ import { MexPairService } from "src/endpoints/mex/mex.pair.service";
import { MexFarmService } from "src/endpoints/mex/mex.farm.service";
import { CachingService, Constants, Lock, GuestCachingWarmer, OriginLogger } from "@multiversx/sdk-nestjs";
import { DelegationLegacyService } from "src/endpoints/delegation.legacy/delegation.legacy.service";
import { PluginService } from "src/common/plugins/plugin.service";
import { SettingsService } from "src/common/settings/settings.service";
import { TokenService } from "src/endpoints/tokens/token.service";
import { IndexerService } from "src/common/indexer/indexer.service";
import { NftService } from "src/endpoints/nfts/nft.service";
import { AccountFilter } from "src/endpoints/accounts/entities/account.filter";
import { TokenType } from "src/common/indexer/entities";
import { TokenDetailed } from "src/endpoints/tokens/entities/token.detailed";
import { DataApiService } from "src/common/data-api/data-api.service";

@Injectable()
export class CacheWarmerService {
Expand All @@ -38,7 +38,6 @@ export class CacheWarmerService {
private readonly identitiesService: IdentitiesService,
private readonly providerService: ProviderService,
private readonly keybaseService: KeybaseService,
private readonly pluginsService: PluginService,
private readonly cachingService: CachingService,
@Inject('PUBSUB_SERVICE') private clientProxy: ClientProxy,
private readonly apiConfigService: ApiConfigService,
Expand All @@ -57,6 +56,7 @@ export class CacheWarmerService {
private readonly indexerService: IndexerService,
private readonly nftService: NftService,
private readonly guestCachingWarmer: GuestCachingWarmer,
private readonly dataApiService: DataApiService,
) {
this.configCronJob(
'handleKeybaseAgainstKeybasePubInvalidations',
Expand Down Expand Up @@ -179,7 +179,7 @@ export class CacheWarmerService {
@Cron(CronExpression.EVERY_MINUTE)
@Lock({ name: 'Current price invalidations', verbose: true })
async handleCurrentPriceInvalidations() {
const currentPrice = await this.pluginsService.getEgldPrice();
const currentPrice = await this.dataApiService.getEgldPrice();
if (currentPrice) {
await this.invalidateKey(CacheInfo.CurrentPrice.key, currentPrice, CacheInfo.CurrentPrice.ttl);
}
Expand Down
6 changes: 3 additions & 3 deletions src/endpoints/network/network.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import { PluginService } from 'src/common/plugins/plugin.service';
import { SmartContractResultService } from '../sc-results/scresult.service';
import { TokenService } from '../tokens/token.service';
import { AccountFilter } from '../accounts/entities/account.filter';
import { DataApiService } from 'src/common/data-api/data-api.service';

@Injectable()
export class NetworkService {
Expand All @@ -35,8 +36,7 @@ export class NetworkService {
private readonly accountService: AccountService,
@Inject(forwardRef(() => TransactionService))
private readonly transactionService: TransactionService,
@Inject(forwardRef(() => PluginService))
private readonly pluginsService: PluginService,
private readonly dataApiService: DataApiService,
private readonly apiService: ApiService,
@Inject(forwardRef(() => StakeService))
private readonly stakeService: StakeService,
Expand Down Expand Up @@ -150,7 +150,7 @@ export class NetworkService {
this.apiConfigService.getDelegationContractAddress(),
'getTotalStakeByType',
),
this.pluginsService.getEgldPrice(),
this.dataApiService.getEgldPrice(),
this.tokenService.getTokenMarketCapRaw(),
]);

Expand Down
Loading

0 comments on commit eed4751

Please sign in to comment.