fix: s3 (#709)

* fix: missing s3 await * get_s3 type err * fix: Decimal type errs * fix: Decimals not json encadable in s3 * fix: s3 * fix: Decimal not json encodable * fix: type err
yearn · Apr 9, 2024 · abb28a4 · abb28a4
1 parent 36babbc
commit abb28a4
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 32 deletions.
diff --git a/scripts/s3.py b/scripts/s3.py
@@ -8,6 +8,8 @@
 import traceback
 import warnings
 from datetime import datetime
+from decimal import Decimal
+from functools import lru_cache
 from time import time
 from typing import Union
 
@@ -40,15 +42,25 @@
 warnings.simplefilter("ignore", BrownieEnvironmentWarning)
 
 METRIC_NAME = "yearn.exporter.apy"
+DEBUG = os.getenv("DEBUG", None)
 
 logs.basicConfig(level=logging.DEBUG)
 logger = logging.getLogger("yearn.apy")
 
 
 async def wrap_vault(
-    vault: Union[VaultV1, VaultV2], samples: ApySamples, aliases: dict, icon_url: str, assets_metadata: dict
+    vault: Union[VaultV1, VaultV2], 
+    samples: ApySamples, 
+    aliases: dict, 
+    icon_url: str, 
+    assets_metadata: dict, 
+    pos: int, 
+    total: int,
 ) -> dict:
-
+    if DEBUG:
+        await _get_debug_lock().acquire()
+    logger.info(f"wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}")
+
     # We don't need results for these right away but they take a while so lets start them now
     inception_fut = asyncio.create_task(contract_creation_block_async(str(vault.vault)))
     apy_fut = asyncio.create_task(get_apy(vault, samples))
@@ -72,7 +84,7 @@ async def wrap_vault(
     if str(vault.vault) in assets_metadata:
         migration = {"available": assets_metadata[str(vault.vault)][1], "address": assets_metadata[str(vault.vault)][2]}
 
-    object = {
+    data = {
         "inception": await inception_fut,
         "address": str(vault.vault),
         "symbol": vault.symbol if hasattr(vault, "symbol") else await ERC20(vault.vault, asynchronous=True).symbol,
@@ -100,9 +112,12 @@ async def wrap_vault(
     }
 
     if chain.id == 1 and any([isinstance(vault, t) for t in [Backscratcher, YveCRVJar]]):
-        object["special"] = True
+        data["special"] = True
 
-    return object
+    logger.info(f"done wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}")
+    if DEBUG:
+        _get_debug_lock().release()
+    return _dedecimal(data)
 
 
 async def get_apy(vault, samples) -> Apy:
@@ -224,14 +239,7 @@ async def _main():
 
     assets_metadata = await get_assets_metadata(registry_v2.vaults)
 
-    data = []
-    total = len(vaults)
-
-    for i, vault in enumerate(vaults):
-        pos = i + 1
-        logger.info(f"wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}")
-        data.append(await wrap_vault(vault, samples, aliases, icon_url, assets_metadata))
-        logger.info(f"done wrapping vault [{pos}/{total}]: {vault.name} {str(vault.vault)}")
+    data = await tqdm_asyncio.gather(*[wrap_vault(vault, samples, aliases, icon_url, assets_metadata, i + 1, len(vaults)) for i, vault in enumerate(vaults)])
 
     if len(data) == 0:
         raise ValueError(f"Data is empty for chain_id: {chain.id}")
@@ -262,7 +270,7 @@ def _export(data, file_name, s3_path):
     with open(file_name, "w+") as f:
         json.dump(data, f)
 
-    if os.getenv("DEBUG", None):
+    if DEBUG:
         return
 
     for item in _get_s3s():
@@ -278,10 +286,10 @@ def _export(data, file_name, s3_path):
 
 def _get_s3s():
     s3s = []
-    aws_buckets = os.environ.get("AWS_BUCKET").split(";")
-    aws_endpoint_urls = os.environ.get("AWS_ENDPOINT_URL").split(";")
-    aws_keys = os.environ.get("AWS_ACCESS_KEY").split(";")
-    aws_secrets = os.environ.get("AWS_ACCESS_SECRET").split(";")
+    aws_buckets = os.environ.get("AWS_BUCKET", "").split(";")
+    aws_endpoint_urls = os.environ.get("AWS_ENDPOINT_URL", "").split(";")
+    aws_keys = os.environ.get("AWS_ACCESS_KEY", "").split(";")
+    aws_secrets = os.environ.get("AWS_ACCESS_SECRET", "").split(";")
 
     for i in range(len(aws_buckets)):
         aws_bucket = aws_buckets[i]
@@ -325,7 +333,7 @@ def _get_export_paths(suffix):
 
 
 def with_monitoring():
-    if os.getenv("DEBUG", None):
+    if DEBUG:
         main()
         return
     from telegram.ext import Updater
@@ -351,3 +359,17 @@ def with_monitoring():
         raise error
     message = f"✅ {export_mode} Vaults API update for {Network.name()} successful!"
     updater.bot.send_message(chat_id=private_group, text=message, reply_to_message_id=ping)
+
+def _dedecimal(dct: dict):
+    """Decimal type cant be json encoded, we make them into floats"""
+    for k, v in dct.items():
+        if isinstance(v, dict):
+            _dedecimal(v)
+        elif isinstance(v, Decimal):
+            dct[k] = float(v)
+    return dct
+
+@lru_cache
+def _get_debug_lock() -> asyncio.Lock:
+    # we use this helper function to ensure the lock is always on the right loop
+    return asyncio.Lock()
diff --git a/yearn/apy/aero.py b/yearn/apy/aero.py
@@ -44,10 +44,10 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples
 
     if end < current_time or total_supply == 0 or rate == 0:
         return Apy("v2:aero_unpopular", gross_apr=0, net_apy=0, fees=fees)
-    pool_price = await magic.get_price(vault.token.address, block=block, sync=False)
+    pool_price = float(await magic.get_price(vault.token.address, block=block, sync=False))
     reward_token = await staking_rewards.rewardToken.coroutine(block_identifier=block) if hasattr(staking_rewards, "rewardToken") else None
     token = reward_token
-    token_price = await magic.get_price(token, block=block, sync=False)
+    token_price = float(await magic.get_price(token, block=block, sync=False))
 
     gross_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (pool_price * (total_supply / 1e18))
 

diff --git a/yearn/apy/curve/rewards.py b/yearn/apy/curve/rewards.py
@@ -39,7 +39,7 @@ async def staking(staking_rewards: Contract, pool_price: int, base_asset_price:
 
     if token and rate:
         # Single reward token
-        token_price = await magic.get_price(token, block=block, sync=False)
+        token_price = float(await magic.get_price(token, block=block, sync=False))
         return (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (
             (pool_price / 1e18) * (total_supply / 1e18) * base_asset_price
         )
@@ -60,7 +60,7 @@ async def staking(staking_rewards: Contract, pool_price: int, base_asset_price:
             except ValueError:
                 token = None
             rate = data.rewardRate / 1e18 if data else 0
-            token_price = await magic.get_price(token, block=block, sync=False) or 0
+            token_price = float(await magic.get_price(token, block=block, sync=False) or 0)
             apr += SECONDS_PER_YEAR * rate * token_price / ((pool_price / 1e18) * (total_supply / 1e18) * token_price)
             queue += 1
             try:
@@ -88,7 +88,7 @@ async def multi(address: str, pool_price: int, base_asset_price: int, block: Opt
             token = None
         if data.periodFinish >= time():
             rate = data.rewardRate / 1e18 if data else 0
-            token_price = await magic.get_price(token, block=block, sync=False) or 0
+            token_price = float(await magic.get_price(token, block=block, sync=False) or 0)
             apr += SECONDS_PER_YEAR * rate * token_price / ((pool_price / 1e18) * (total_supply / 1e18) * token_price)
         queue += 1
         try:

diff --git a/yearn/apy/curve/simple.py b/yearn/apy/curve/simple.py
@@ -236,7 +236,7 @@ async def calculate_simple(vault, gauge: Gauge, samples: ApySamples) -> Apy:
             if period_finish < current_time:
                 reward_apr = 0
             else:
-                reward_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / ((pool_price / 1e18) * (total_supply / 1e18) * base_asset_price)
+                reward_apr = (SECONDS_PER_YEAR * (rate / 1e18) * float(token_price)) / ((float(pool_price) / 1e18) * (total_supply / 1e18) * float(base_asset_price))
     else:
         reward_apr = 0
 
@@ -519,7 +519,7 @@ async def _get_reward_apr(self, cvx_strategy, cvx_booster, base_asset_price, poo
                     virtual_rewards_pool.totalSupply.coroutine(),
                 )
 
-                reward_apr = (reward_rate * SECONDS_PER_YEAR * reward_token_price) / (base_asset_price * (pool_price / 1e18) * total_supply)
+                reward_apr = (reward_rate * SECONDS_PER_YEAR * float(reward_token_price)) / (float(base_asset_price) * (float(pool_price) / 1e18) * total_supply)
                 convex_reward_apr += reward_apr
 
         return convex_reward_apr

diff --git a/yearn/apy/staking_rewards.py b/yearn/apy/staking_rewards.py
@@ -14,10 +14,11 @@ async def get_staking_rewards_apr(vault, samples: ApySamples):
         return 0
 
     vault_address = str(vault.vault)
-    if vault_address not in vault.registry.staking_pools:
+    staking_pools = await vault.registry.staking_pools
+    if vault_address not in staking_pools:
         return 0
 
-    staking_pool = await Contract.coroutine(vault.registry.staking_pools[vault_address])
+    staking_pool = await Contract.coroutine(staking_pools[vault_address])
     if await staking_pool.periodFinish.coroutine() < now:
         return 0
 

diff --git a/yearn/apy/velo.py b/yearn/apy/velo.py
@@ -43,11 +43,11 @@ async def staking(vault: "Vault", staking_rewards: Contract, samples: ApySamples
 
     if end < current_time or total_supply == 0 or rate == 0:
         return Apy("v2:velo_unpopular", gross_apr=0, net_apy=0, fees=fees)
-    else:
-        pool_price = await magic.get_price(vault.token.address, block=block, sync=False)
-    reward_token = await staking_rewards.rewardToken.coroutine(block_identifier=block) if hasattr(staking_rewards, "rewardToken") else None
+
+    pool_price = float(await magic.get_price(vault.token.address, block=block, sync=False))
+    reward_token = await staking_rewards.rewardToken.coroutine(block_identifier=block)
     token = reward_token
-    token_price = await magic.get_price(token, block=block, sync=False)
+    token_price = float(await magic.get_price(token, block=block, sync=False))
 
     gross_apr = (SECONDS_PER_YEAR * (rate / 1e18) * token_price) / (pool_price * (total_supply / 1e18))