Skip to content

Commit

Permalink
fix(s3): use uniform paths for base and loanscan api (yearn#106)
Browse files Browse the repository at this point in the history
  • Loading branch information
@nymmrx
nymmrx authored Aug 4, 2021
1 parent 98c68b7 commit 9742602
Show file tree
Hide file tree
Showing 2 changed files with 47 additions and 70 deletions.
39 changes: 16 additions & 23 deletions scripts/s3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,18 @@
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger("yearn.apy")

def wrap_vault(vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dict, icon_url: str, assets_metadata: dict) -> dict:

def wrap_vault(
vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dict, icon_url: str, assets_metadata: dict
) -> dict:
apy = vault.apy(samples)

new = False
if isinstance(vault, VaultV2):
harvests = [harvest for strategy in vault.strategies for harvest in strategy.harvests]
if len(harvests) < 4:
new = True

if isinstance(vault, VaultV1):
strategies = [
{
Expand All @@ -55,7 +58,7 @@ def wrap_vault(vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dic
strategies = [{"address": str(strategy.strategy), "name": strategy.name} for strategy in vault.strategies]

inception = contract_creation_block(str(vault.vault))

token_alias = aliases[str(vault.token)]["symbol"] if str(vault.token) in aliases else vault.token.symbol()
vault_alias = token_alias

Expand All @@ -64,11 +67,7 @@ def wrap_vault(vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dic
migration = None

if str(vault.vault) in assets_metadata:
migration = {
"available": assets_metadata[str(vault.vault)][1],
"address": assets_metadata[str(vault.vault)][2]
}

migration = {"available": assets_metadata[str(vault.vault)][1], "address": assets_metadata[str(vault.vault)][2]}

object = {
"inception": inception,
Expand All @@ -83,7 +82,7 @@ def wrap_vault(vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dic
"address": str(vault.token),
"decimals": vault.token.decimals() if hasattr(vault.token, "decimals") else None,
"display_name": token_alias,
"icon": icon_url % str(vault.token)
"icon": icon_url % str(vault.token),
},
"tvl": dataclasses.asdict(tvl),
"apy": dataclasses.asdict(apy),
Expand All @@ -95,14 +94,15 @@ def wrap_vault(vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dic
"emergency_shutdown": vault.vault.emergencyShutdown() if hasattr(vault.vault, "emergencyShutdown") else False,
"updated": int(time()),
"migration": migration,
"new": new
"new": new,
}

if any([isinstance(vault, t) for t in [Backscratcher, YveCRVJar]]):
object["special"] = True

return object


def get_assets_metadata(vault_v2: list) -> dict:
registry_v2_adapter = Contract(web3.ens.resolve("lens.ychad.eth"))
addresses = [str(vault.vault) for vault in vault_v2]
Expand Down Expand Up @@ -168,33 +168,26 @@ def main():
aws_secret = os.environ.get("AWS_ACCESS_SECRET")
aws_bucket = os.environ.get("AWS_BUCKET")

s3 = boto3.client(
"s3",
aws_access_key_id=aws_key,
aws_secret_access_key=aws_secret
)
s3 = boto3.client("s3", aws_access_key_id=aws_key, aws_secret_access_key=aws_secret)

s3.upload_file(
os.path.join(out, vault_api_all),
aws_bucket,
vault_api_all,
ExtraArgs={
'ContentType': "application/json",
'CacheControl': "max-age=1800"
}
ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"},
)

s3.upload_file(
os.path.join(out, vault_api_experimental),
aws_bucket,
vault_api_experimental,
ExtraArgs={
'ContentType': "application/json",
'CacheControl': "max-age=1800"
}
ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"},
)


telegram_users_to_alert = ["@nymmrx", "@x48114", "@dudesahn"]


def with_monitoring():
from telegram.ext import Updater

Expand Down
78 changes: 31 additions & 47 deletions scripts/s3_loanscan.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import warnings
import traceback
from dotenv import find_dotenv, load_dotenv

load_dotenv(find_dotenv())


Expand All @@ -37,19 +38,14 @@ def get_assets_metadata(vault_v2: list) -> dict:
def get_formatted_lend_rates(vault: VaultV2, samples: ApySamples) -> list:
apy = vault.apy(samples)
lend_rate_apy = apy.net_apy
lend_rate_apr = ((apy.net_apy + 1)**(1/365) - 1) * 365
lend_rate_apr = ((apy.net_apy + 1) ** (1 / 365) - 1) * 365
if apy.type == 'crv':
return [{
"apr": lend_rate_apr,
"apy": lend_rate_apy,
"tokenSymbol": Contract(curve_pool_token_address).symbol()
} for curve_pool_token_address in curve.get_underlying_coins(vault.token)]
return [
{"apr": lend_rate_apr, "apy": lend_rate_apy, "tokenSymbol": Contract(curve_pool_token_address).symbol()}
for curve_pool_token_address in curve.get_underlying_coins(vault.token)
]
else:
return [{
"apr": lend_rate_apr,
"apy": lend_rate_apy,
"tokenSymbol": vault.token.symbol()
}]
return [{"apr": lend_rate_apr, "apy": lend_rate_apy, "tokenSymbol": vault.token.symbol()}]


def write_json(json_dict: dict, path: str):
Expand Down Expand Up @@ -77,20 +73,15 @@ def main():
yveCrv_symbol = yveCrvVault.vault.symbol()
yveCrv_lend_rates = get_formatted_lend_rates(yveCrvVault, samples)
loanscan_vault_symbols.append(yveCrv_symbol)
loanscan_vault_json.append({
"symbol": yveCrv_symbol,
"lendRates": yveCrv_lend_rates
})
loanscan_vault_json.append({"symbol": yveCrv_symbol, "lendRates": yveCrv_lend_rates})
# Don't simplify above this comment
except Exception as yveCrvException:
logger.info(
f'failed to reduce yveCrv lendRate, {str(yveCrvVault.vault)} {yveCrvVault}')
logger.info(f'failed to reduce yveCrv lendRate, {str(yveCrvVault.vault)} {yveCrvVault}')
logger.error(yveCrvException)

for vault in registry_v2.vaults:
try:
vault_not_endorsed = not (
hasattr(vault, "is_endorsed") and vault.is_endorsed)
vault_not_endorsed = not (hasattr(vault, "is_endorsed") and vault.is_endorsed)
if vault_not_endorsed:
continue

Expand All @@ -106,18 +97,15 @@ def main():
vault_symbol = vault.vault.symbol()
vault_lend_rates = get_formatted_lend_rates(vault, samples)
loanscan_vault_symbols.append(vault_symbol)
loanscan_vault_json.append({
"symbol": vault_symbol,
"lendRates": vault_lend_rates
})
loanscan_vault_json.append({"symbol": vault_symbol, "lendRates": vault_lend_rates})
# Don't simplify above this comment
except Exception as error:
logger.info(
f'failed to reduce loanscan lendRate for vault {str(vault.vault)} {vault}')
logger.info(f'failed to reduce loanscan lendRate for vault {str(vault.vault)} {vault}')
logger.error(error)

out_path = "generated"
loanscan_path = os.path.join(out_path, "loanscan")
base_path_s3 = os.path.join("v1", "chains", "1", "loanscan")
loanscan_path = os.path.join(out_path, base_path_s3)
os.makedirs(out_path, exist_ok=True)
if os.path.isdir(loanscan_path):
shutil.rmtree(loanscan_path)
Expand All @@ -126,56 +114,52 @@ def main():
write_json(loanscan_vault_symbols, os.path.join(loanscan_path, "manifest"))
write_json(loanscan_vault_json, os.path.join(loanscan_path, "all"))
for loanscan_vault in loanscan_vault_json:
write_json({
"lendRates": loanscan_vault["lendRates"]
}, os.path.join(loanscan_path, loanscan_vault["symbol"]))
write_json({"lendRates": loanscan_vault["lendRates"]}, os.path.join(loanscan_path, loanscan_vault["symbol"]))

aws_bucket = os.environ.get("AWS_BUCKET")
s3 = boto3.client(
"s3",
aws_access_key_id=os.environ.get("AWS_ACCESS_KEY"),
aws_secret_access_key=os.environ.get("AWS_ACCESS_SECRET")
aws_secret_access_key=os.environ.get("AWS_ACCESS_SECRET"),
)

for loanscan_json_filename in os.listdir(loanscan_path):
try:
file_path = os.path.join(loanscan_path, loanscan_json_filename)
file_path_s3 = os.path.join(base_path_s3, loanscan_json_filename)
s3.upload_file(
os.path.join(file_path),
file_path,
aws_bucket,
os.path.join("v1", "chains", "1", "loanscan", loanscan_json_filename),
ExtraArgs={
'ContentType': "application/json",
'CacheControl': "max-age=1800"
}
file_path_s3,
ExtraArgs={'ContentType': "application/json", 'CacheControl': "max-age=1800"},
)
except Exception as error:
logger.info(f'failed to upload {file_path} to s3')
logger.error(error)


telegram_users_to_alert = ["@nymmrx", "@x48114", "@dudesahn"]


def with_monitoring():
from telegram.ext import Updater

private_group = os.environ.get('TG_YFIREBOT_GROUP_INTERNAL')
public_group = os.environ.get('TG_YFIREBOT_GROUP_EXTERNAL')
updater = Updater(os.environ.get('TG_YFIREBOT'))
now = datetime.now()
message = f"`[{now}]`\n⚙️ Loanscan API is updating..."
ping = updater.bot.send_message(
chat_id=private_group, text=message, parse_mode="Markdown")
message = f"`[{now}]`\n⚙️ API is updating..."
ping = updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown")
ping = ping.message_id
try:
main()
except Exception as error:
tb = traceback.format_exc()
now = datetime.now()
message = f"`[{now}]`\n🔥 Loanscan API update failed!\n```\n{tb}\n```"
updater.bot.send_message(
chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping)
updater.bot.send_message(chat_id=public_group,
text=message, parse_mode="Markdown")
tags = " ".join(telegram_users_to_alert)
message = f"`[{now}]`\n🔥 API update failed!\n```\n{tb}\n```\n{tags}"
updater.bot.send_message(chat_id=private_group, text=message, parse_mode="Markdown", reply_to_message_id=ping)
updater.bot.send_message(chat_id=public_group, text=message, parse_mode="Markdown")
raise error
message = "✅ Loanscan API update successful!"
updater.bot.send_message(
chat_id=private_group, text="✅ Loanscan API update successful!", reply_to_message_id=ping)
message = "✅ API update successful!"
updater.bot.send_message(chat_id=private_group, text="✅ API update successful!", reply_to_message_id=ping)

0 comments on commit 9742602

Please sign in to comment.