Describe the bug
I am constantly getting 500 internal error responses for get_vault_assets_balance. Other times I am also receiving:
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
To Reproduce
Please see below
Expected behavior
I am trying to make a snapshot of our Fireblocks account for backup and also accounting purposes. I am trying to iterate over each vault and its assets and gathering balances for each.
Versions (please complete the following information):
Python Version: 3.12.1
fireblocks-sdk version: fireblocks_sdk==2.5.1
Additional context
Here is an excerpt of my related code snippets and logs:
def catch_and_retry_fireblocks_exception(func):
@functools.wraps(func)
def wrapper(self, *args, **kwargs):
max_retries = 3
retries = 0
while retries < max_retries:
try:
return func(self, *args, **kwargs)
except FireblocksApiException as e:
self.logger.error(
f"[Fireblocks] Error occurred for function {func.__name__}: {e}"
)
self.logger.error(f"[Fireblocks] Arguments: {args}, {kwargs}")
# Handle the exception as needed
retries += 1
if retries < max_retries:
# Wait for 1 second before retrying
time.sleep(0.1)
self.logger.info(f"[Fireblocks] Retrying {func.__name__}...")
else:
self.logger.warning(
f"[Fireblocks] Maximum retries reached. Skipping {func.__name__}."
)
return wrapper
class FireblocksSDKWrapper:
def __init__(self, api_key: str, private_key: str):
self.fireblocks = FireblocksSDK(api_key=api_key, private_key=private_key)
self.logger = Logger(name=__name__, level="INFO").get_logger()
@catch_and_retry_fireblocks_exception
def get_vault_accounts_paginated(self):
next_page = True
paged_filter = PagedVaultAccountsRequestFilters()
accounts = []
while next_page:
vaults = self.fireblocks.get_vault_accounts_with_page_info(paged_filter)
accounts = vaults["accounts"]
accounts.extend(accounts)
# Paginate through accounts
if vaults.get("paging").get("after") is None:
next_page = False
else:
paged_filter.after = vaults.get("paging").get("after")
return accounts
@catch_and_retry_fireblocks_exception
def get_internal_wallets(self):
return self.fireblocks.get_internal_wallets()
@catch_and_retry_fireblocks_exception
def get_deposit_addresses(self, vault_account_id: typing.Any, asset_id: typing.Any):
return self.fireblocks.get_deposit_addresses(vault_account_id, asset_id)
@catch_and_retry_fireblocks_exception
def get_vault_assets_balance(
self,
account_name_prefix: typing.Any | None = None,
account_name_suffix: typing.Any | None = None,
):
return self.fireblocks.get_vault_assets_balance(
account_name_prefix,
account_name_suffix,
)
@dataclasses.dataclass
class FireblocksAccountingLoader(ExchangeLoaderBase):
fireblocks: typing.Optional[FireblocksSDKWrapper] = dataclasses.field(init=True)
vault_accounts: typing.List[dict] = dataclasses.field(init=False)
internal_wallets: typing.List[dict] = dataclasses.field(init=False)
accounts: typing.List[dict] = dataclasses.field(init=False)
logger: logging.Logger = dataclasses.field(
default_factory=lambda: logging.Logger(__name__)
)
def __post_init__(self):
self.vault_accounts = self.fireblocks.get_vault_accounts_paginated()
self.internal_wallets = self.fireblocks.get_internal_wallets()
self.accounts = self.vault_accounts # + self.internal_wallets
def fetch_accounts(self, utc_timestamp: str) -> pd.DataFrame:
df = pd.DataFrame(self.accounts)
# explode the assets column
df = df.explode("assets")
# Cast nan assets to {}
df.assets = df.assets.fillna({i: {} for i in df.index})
# create new columns for each key in the dictionaries
df["asset_id"] = df["assets"].apply(lambda x: x["id"] if "id" in x else None)
df["asset_total"] = df["assets"].apply(
lambda x: x["total"] if "total" in x else None
)
df["asset_balance"] = df["assets"].apply(
lambda x: x["balance"] if "balance" in x else None
)
df["account_type"] = df["name"].apply(
lambda x: (
"internal_wallet" if x == "Gas Station Wallet" else "vault_account"
)
)
# Cast id to int
df["id"] = df["id"].astype(int)
# Remove "Network Deposits" vault accounts
df = df[df.name != "Network Deposits"]
# Add the timestamp to the dataframe
df["timestamp"] = utc_timestamp
# drop the original assets column
df = df.drop("assets", axis=1)
# Reset index
df = df.reset_index(drop=True)
# Fetch asset_id to a dict
# Get the vault account asset information
for row, col in df.iterrows():
# Skip if the account is an internal wallet
if col.account_type == "internal_wallet":
# Get an index of the internal wallet
internal_wallet_index = [
i for i, x in enumerate(self.internal_wallets) if x["id"] == col.id
][0]
asset_info = self.internal_wallets[internal_wallet_index]["assets"]
elif col.account_type == "vault_account" and col.asset_id is not None:
asset_info = self.fireblocks.get_deposit_addresses(
vault_account_id=col.id,
asset_id=col.asset_id,
)
else:
asset_info = []
# Restructure the asset_info into a dictionary
asset_dict = {}
for asset in asset_info:
asset_dict[asset["address"]] = {
"asset_id": asset["assetId"] if "assetId" in asset else None,
"address": asset["address"] if "address" in asset else None,
"tag": asset["tag"] if "tag" in asset else None,
"description": (
asset["description"] if "description" in asset else None
),
"type": asset["type"] if "type" in asset else None,
"addressFormat": (
asset["addressFormat"] if "addressFormat" in asset else None
),
"legacyAddress": (
asset["legacyAddress"] if "legacyAddress" in asset else None
),
"bip44AddressIndex": (
asset["bip44AddressIndex"]
if "bip44AddressIndex" in asset
else None
),
}
# Add asset_info as a column to the dataframe being iterated over
df.loc[row, "addresses"] = [[asset_dict]]
# Get additional balance information
df_assets = pd.DataFrame()
for vault_name in df.name.unique():
# Skip if the account is an internal wallet
if vault_name == "Gas Station Wallet":
continue
asset_balances = pd.DataFrame(
self.fireblocks.get_vault_assets_balance(vault_name)
)
asset_balances = asset_balances.rename(columns={"id": "asset_id"})
asset_balances["name"] = vault_name
df_assets = pd.concat([df_assets, asset_balances], axis=0)
# Merge the asset balances with the dataframe
df_merged = pd.merge(df, df_assets, on=["name", "asset_id"])
# Calculate prices
asset_ids = df_merged.asset_id.unique()
asset_prices = {}
for asset_id in asset_ids:
try:
asset_id_split = asset_id.split("_")[0]
asset_prices[asset_id] = self.calculate_usd_price(
asset=asset_id_split,
exchange=ccxt.bitmart(), # Using bitmart since only they have RND
)
except Exception as e: # pylint: disable=broad-except
self.logger.info(
"[Fireblocks] Could not fetch price for %s (error: %s)",
asset_id,
e,
)
continue
for row, col in df_merged.iterrows():
try:
asset_price = asset_prices[col.asset_id]
df_merged.loc[row, "usd_price"] = asset_prices[col.asset_id]
df_merged.loc[row, "usd_value"] = (
col.asset_total * asset_price if asset_price is not None else None
)
except Exception as e: # pylint: disable=broad-except
self.logger.info(
"[Fireblocks] Could not set price for %s: %s)", col.asset_id, e
)
continue
return df_merged
def to_dataframe(self, utc_timestamp: pd.Timestamp = None):
return self.fetch_accounts(
utc_timestamp=(
pd.Timestamp.utcnow() if utc_timestamp is None else utc_timestamp
)
)
2024-03-07 10:56:19,315 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 10:56:19,426 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:08:53,224 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:08:53,327 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:11:35,880 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:11:35,984 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:14:23,272 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:14:23,378 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:17:10,825 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:17:10,932 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:19:52,531 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:19:52,640 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:22:33,295 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:22:33,401 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:25:17,782 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:25:17,890 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
2024-03-07 11:28:00,107 - rand_treasury_monitoring.utils.fireblocks - ERROR - [Fireblocks] Error occurred for function get_vault_assets_balance: Got an error from fireblocks server: 500
2024-03-07 11:28:00,211 - rand_treasury_monitoring.utils.fireblocks - INFO - [Fireblocks] Retrying get_vault_assets_balance...
Exception has occurred: ConnectionError (note: full exception trace is shown but execution is paused at: <module>)
('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
During handling of the above exception, another exception occurred:
File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/utils/fireblocks.py", line 74, in get_vault_assets_balance
return self.fireblocks.get_vault_assets_balance(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/utils/fireblocks.py", line 17, in wrapper
return func(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/data/loaders.py", line 316, in fetch_accounts
self.fireblocks.get_vault_assets_balance(vault_name)
File "/Users/adr/Dev/Rand-Network/treasury-balance-service/rand_treasury_monitoring/data/loaders.py", line 357, in to_dataframe
return self.fetch_accounts(
^^^^^^^^^^^^^^^^^^^^
File "/var/folders/w6/dh79cy2x0sjdkb2y4vrs8kq00000gn/T/ipykernel_18952/3364672574.py", line 1, in <module> (Current frame)
loader.to_dataframe()
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
Describe the bug I am constantly getting 500 internal error responses for
get_vault_assets_balance. Other times I am also receiving:To Reproduce Please see below
Expected behavior I am trying to make a snapshot of our Fireblocks account for backup and also accounting purposes. I am trying to iterate over each vault and its assets and gathering balances for each.
Versions (please complete the following information):
Additional context Here is an excerpt of my related code snippets and logs: