diff --git a/monitoring.yaml b/monitoring.yaml index cf3ef60..3fc315f 100644 --- a/monitoring.yaml +++ b/monitoring.yaml @@ -26,23 +26,23 @@ protocols: - name: "Price Per Share" description: "USD3 PPS decrease (any drop vs cached prior, CRITICAL) and sUSD3 PPS decrease (HIGH)" - name: "TVL" - description: "Absolute TVL change >=15% for USD3 or sUSD3" + description: "Absolute TVL change >=15% for USD3 or sUSD3 using a baseline no older than 3 hours" - name: "Junior Buffer Ratio" - description: "sUSD3 backing <15% of deployed credit" + description: "sUSD3 backing <15% of deployed credit; re-armed after a monitoring gap >3 hours" - name: "USD3 Overcollateralization" - description: "OC <111% (HIGH) or <106% (CRITICAL)" + description: "OC <111% (HIGH) or <106% (CRITICAL); re-armed after a monitoring gap >3 hours" - name: "Insurance Fund Outflow" description: "waUSDC outflow >=$50k since prior run" - name: "Withdraw Liquidity" - description: "USD3 availableWithdrawLimit < $4M" + description: "USD3 availableWithdrawLimit < $4M; re-armed after a monitoring gap >3 hours" - name: "Vault Shutdown" - description: "Alert-once when isShutdown() transitions to true" + description: "Alert-once when isShutdown() is true; re-armed after a monitoring gap >3 hours" - name: "Debt Cap" description: "Any change to ProtocolConfig.getDebtCap()" - name: "Nominal sUSD3 Backing Floor" - description: "Any floor change; alert-once when floor > sUSD3 backing" + description: "Any floor change; floor-breach dedupe re-armed after a monitoring gap >3 hours" - name: "Protocol Pause" - description: "Alert-once when ProtocolConfig IS_PAUSED flips true" + description: "Alert-once when IS_PAUSED is true; re-armed after a monitoring gap >3 hours" - name: "Borrower Default Watch" description: "Envio-backed MorphoCredit borrower watch; MEDIUM alerts when unpaid obligations become delinquent after grace or reach default" - name: "Timelock" @@ -85,7 +85,7 @@ protocols: - name: "Withdrawable Liquidity" description: "Total withdrawable liquidity across cUSD assets < $15M" - name: "Large cUSD Mints" - description: "totalSupply delta >5% vs cached prior" + description: "totalSupply delta >5% vs a cached prior no older than 36 hours" - name: "Timelock" description: "CAP TimelockController events via Envio" - name: "Safe Multisig" @@ -171,19 +171,19 @@ protocols: - protocols/infinifi/main.py monitors: - name: "Liquid Reserves" - description: "Liquid reserves < $8M" + description: "Liquid reserves < $8M; crossing detection re-armed after a monitoring gap >3 hours" - name: "Backing Per iUSD" - description: "totalTVL / iUSD supply < 0.999" + description: "totalTVL / iUSD supply < 0.999; re-armed after a monitoring gap >3 hours" - name: "Redemption Pressure" - description: "Pending redemptions / liquid reserves >80%" + description: "Pending redemptions / liquid reserves >80%; re-armed after a monitoring gap >3 hours" - name: "Large iUSD Mints" - description: "totalSupply delta >5% vs cached prior" + description: "totalSupply delta >5% vs a cached prior no older than 3 hours" - name: "Farm Allocation Shift" description: "Farm allocation ratio change >30% vs prior run (farms <1% of TVL excluded)" - name: "Farm Activation" description: "Farm previously at 0 ratio moves above 3% of TVL" - name: "Junior TVL Coverage" - description: "Junior TVL <50% of risky farm exposure" + description: "Junior TVL <50% of risky farm exposure; re-armed after a monitoring gap >3 hours" - name: "Timelock" description: "Longtimelock and Shorttimelock events via Envio" @@ -245,7 +245,7 @@ protocols: - name: "Price Per Share" description: "Any PPS decrease" - name: "TVL" - description: "Absolute TVL change >=15%" + description: "Absolute TVL change >=15% using a baseline no older than 3 hours" - name: "Unrealized Losses" description: "Any non-zero on-chain unrealized losses; >=0.5% of pool assets via subgraph" - name: "Withdrawal Queue" @@ -259,7 +259,7 @@ protocols: - name: "Unknown Collateral" description: "Collateral assets not in risk mapping" - name: "Pool Delegate Cover" - description: "Cover balance drops to $0 or decreases vs cached prior" + description: "Cover balance drops to $0 or decreases vs cached prior; zero state re-armed after >3 hours" - name: "Timelock" description: "Maple GovernorTimelock events via Envio" - name: "Safe Multisig" @@ -388,13 +388,13 @@ protocols: - name: "srUSDe Exchange Rate" description: "srUSDe convertToAssets decrease vs cached prior" - name: "Senior Coverage Ratio" - description: "Senior coverage <105%" + description: "Senior coverage <105%; re-armed after a monitoring gap >36 hours" - name: "Junior Tranche Drain" - description: "jrUSDe totalAssets drop >=15%" + description: "jrUSDe totalAssets drop >=15% using a baseline no older than 36 hours" - name: "Strategy Balance" - description: "sUSDe strategy balance / total deposits drops >=20%" + description: "sUSDe strategy balance / total deposits drops >=20% using a baseline no older than 36 hours" - name: "TVL" - description: "Total deposits change >=15%" + description: "Total deposits change >=15% using a baseline no older than 36 hours" - name: "sUSDe Dependency" description: "sUSDe vault rate monotonicity and cooldown duration changes" - name: "Timelock" @@ -424,7 +424,7 @@ protocols: - name: "Oracle Divergence" description: "Continuous Oracle vs Chainlink differ by >0.5%" - name: "Supply Changes" - description: "Total supply change >10% vs prior run" + description: "Total supply change >10% vs a prior observation no older than 3 hours" - name: "Oracle Staleness" description: "Latest checkpoint effectiveAt >4 days old" - name: "Stablecoin Price" @@ -439,13 +439,13 @@ protocols: - protocols/usdai/large_mints.py monitors: - name: "Backing Invariant" - description: "totalSupply + bridgedSupply - PYUSD balance >= 100 USDai" + description: "totalSupply + bridgedSupply - PYUSD balance >= 100 USDai; re-armed after a gap >3 hours" - name: "Loan Activity" description: "Total verified principal change >1% vs prior run" - name: "Legacy Loan Expiry" description: "Hardcoded NVIDIA H200s legacy loan past 2028-07-27" - name: "Large Mints" - description: "totalSupply delta >5% via large_mints.py" + description: "totalSupply delta >5% via large_mints.py using a baseline no older than 3 hours" - name: "Safe Multisigs" description: "USDai and sUSDai Admin Safe queues" diff --git a/protocols/3jane/README.md b/protocols/3jane/README.md index a312080..2ad1d28 100644 --- a/protocols/3jane/README.md +++ b/protocols/3jane/README.md @@ -45,6 +45,13 @@ | Borrower delinquent/default watch | New milestone: delinquent, ≤14d, ≤7d, ≤3d, ≤1d, default | MEDIUM | | Monitoring run failure | Uncaught exception in `main()` | LOW | +## Cache Freshness + +TVL delta baselines expire after 3 hours and initialize from the next valid observation. Current-state and threshold +dedupe for junior buffer, USD3 OC, withdraw liquidity, vault shutdown, nominal-floor breach, and protocol pause is +re-armed after the same monitoring gap. PPS, insurance-fund, governance-value, and borrower-event history does not +expire. + ## Borrower default watch Set `ENVIO_GRAPHQL_URL` to the 3Jane Envio GraphQL endpoint to enable proactive borrower monitoring. Without this env var, the borrower default watch is skipped and all other 3Jane checks continue normally. diff --git a/protocols/3jane/main.py b/protocols/3jane/main.py index 691ce06..03bbc24 100644 --- a/protocols/3jane/main.py +++ b/protocols/3jane/main.py @@ -34,7 +34,14 @@ from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_path, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + HOURLY_CACHE_STALE_AFTER_SECONDS, + cache_path, + get_fresh_last_value_for_key_from_file, + get_last_value_for_key_from_file, + write_last_value_to_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.formatting import format_duration, format_usd from utils.logger import get_logger @@ -168,9 +175,23 @@ def set_cache_value(key: str, value: int | float) -> None: write_last_value_to_file(CACHE_FILENAME, key, value) +def get_fresh_cache_value(key: str) -> float: + """Read an hourly cache value, returning zero when its observation is stale.""" + val = get_fresh_last_value_for_key_from_file(CACHE_FILENAME, key, HOURLY_CACHE_STALE_AFTER_SECONDS) + try: + return float(val) + except (ValueError, TypeError): + return 0.0 + + +def set_fresh_cache_value(key: str, value: int | float) -> None: + """Write an hourly cache value and its observation timestamp.""" + write_last_value_with_timestamp_to_file(CACHE_FILENAME, key, value) + + def _get_alerted_value(cache_key: str) -> float: """Read the last alerted value for a threshold alert (-1 = none outstanding).""" - raw_cached = get_last_value_for_key_from_file(CACHE_FILENAME, cache_key) + raw_cached = get_fresh_last_value_for_key_from_file(CACHE_FILENAME, cache_key, HOURLY_CACHE_STALE_AFTER_SECONDS) try: return float(raw_cached) if isinstance(raw_cached, str) else -1.0 except ValueError: @@ -199,18 +220,20 @@ def should_alert_value_drop(cache_key: str, value: float, threshold: float) -> b return False cached = _get_alerted_value(cache_key) - return not 0 <= cached <= value + should_alert = not 0 <= cached <= value + if not should_alert: + set_fresh_cache_value(cache_key, cached) + return should_alert def mark_alerted_value(cache_key: str, value: float) -> None: """Record the value a threshold alert fired for; call after send_alert() returns.""" - set_cache_value(cache_key, value) + set_fresh_cache_value(cache_key, value) def clear_alerted_value(cache_key: str) -> None: """Clear an outstanding threshold alert so the next breach alerts again.""" - if _get_alerted_value(cache_key) >= 0: - set_cache_value(cache_key, -1) + set_fresh_cache_value(cache_key, -1) def _as_bool(value: Any) -> bool: @@ -544,7 +567,7 @@ def check_tvl(usd3_tvl: float, susd3_tvl: float) -> None: susd3_tvl: Current sUSD3 totalAssets in USD3 terms. """ # --- USD3 TVL --- - previous_usd3_tvl = get_cache_value(CACHE_KEY_USD3_TVL) + previous_usd3_tvl = get_fresh_cache_value(CACHE_KEY_USD3_TVL) logger.info("USD3 TVL: %s (previous: %s)", format_usd(usd3_tvl), format_usd(previous_usd3_tvl)) if previous_usd3_tvl > 0: @@ -559,11 +582,10 @@ def check_tvl(usd3_tvl: float, susd3_tvl: float) -> None: ) send_alert(Alert(AlertSeverity.LOW, message, PROTOCOL)) - if usd3_tvl != previous_usd3_tvl: - set_cache_value(CACHE_KEY_USD3_TVL, usd3_tvl) + set_fresh_cache_value(CACHE_KEY_USD3_TVL, usd3_tvl) # --- sUSD3 TVL --- - previous_susd3_tvl = get_cache_value(CACHE_KEY_SUSD3_TVL) + previous_susd3_tvl = get_fresh_cache_value(CACHE_KEY_SUSD3_TVL) logger.info("sUSD3 TVL: %s (previous: %s)", format_usd(susd3_tvl), format_usd(previous_susd3_tvl)) if previous_susd3_tvl > 0: @@ -579,8 +601,7 @@ def check_tvl(usd3_tvl: float, susd3_tvl: float) -> None: ) send_alert(Alert(AlertSeverity.LOW, message, PROTOCOL)) - if susd3_tvl != previous_susd3_tvl: - set_cache_value(CACHE_KEY_SUSD3_TVL, susd3_tvl) + set_fresh_cache_value(CACHE_KEY_SUSD3_TVL, susd3_tvl) def check_junior_buffer(susd3_backing: float, deployed_credit: float) -> None: @@ -765,7 +786,7 @@ def check_vault_shutdown(client, usd3_vault, susd3_vault) -> None: # type: igno logger.info("Vault shutdown — USD3: %s, sUSD3: %s", usd3_shutdown, susd3_shutdown) # Alert once on USD3 shutdown - previous_usd3_shutdown = get_cache_value(CACHE_KEY_SHUTDOWN_USD3) + previous_usd3_shutdown = get_fresh_cache_value(CACHE_KEY_SHUTDOWN_USD3) if usd3_shutdown and previous_usd3_shutdown == 0: message = ( f"🚨 *3Jane USD3 Vault SHUTDOWN*\n" @@ -773,11 +794,10 @@ def check_vault_shutdown(client, usd3_vault, susd3_vault) -> None: # type: igno f"🔗 [USD3](https://etherscan.io/address/{USD3_ADDRESS})" ) send_alert(Alert(AlertSeverity.CRITICAL, message, PROTOCOL)) - if float(usd3_shutdown) != previous_usd3_shutdown: - set_cache_value(CACHE_KEY_SHUTDOWN_USD3, float(usd3_shutdown)) + set_fresh_cache_value(CACHE_KEY_SHUTDOWN_USD3, float(usd3_shutdown)) # Alert once on sUSD3 shutdown - previous_susd3_shutdown = get_cache_value(CACHE_KEY_SHUTDOWN_SUSD3) + previous_susd3_shutdown = get_fresh_cache_value(CACHE_KEY_SHUTDOWN_SUSD3) if susd3_shutdown and previous_susd3_shutdown == 0: message = ( f"🚨 *3Jane sUSD3 Vault SHUTDOWN*\n" @@ -785,8 +805,7 @@ def check_vault_shutdown(client, usd3_vault, susd3_vault) -> None: # type: igno f"🔗 [sUSD3](https://etherscan.io/address/{SUSD3_ADDRESS})" ) send_alert(Alert(AlertSeverity.CRITICAL, message, PROTOCOL)) - if float(susd3_shutdown) != previous_susd3_shutdown: - set_cache_value(CACHE_KEY_SHUTDOWN_SUSD3, float(susd3_shutdown)) + set_fresh_cache_value(CACHE_KEY_SHUTDOWN_SUSD3, float(susd3_shutdown)) def check_debt_cap(client) -> None: # type: ignore[no-untyped-def] @@ -862,7 +881,7 @@ def check_nominal_backing_floor(nominal_floor: float, susd3_backing: float) -> N # --- Alert-once on breach transition (floor > backing) --- breach = nominal_floor > susd3_backing and nominal_floor > 0 - previous_breach = get_cache_value(CACHE_KEY_FLOOR_BREACH) + previous_breach = get_fresh_cache_value(CACHE_KEY_FLOOR_BREACH) if breach and previous_breach == 0: shortfall = nominal_floor - susd3_backing message = ( @@ -873,8 +892,7 @@ def check_nominal_backing_floor(nominal_floor: float, susd3_backing: float) -> N f"🔗 [sUSD3](https://etherscan.io/address/{SUSD3_ADDRESS})" ) send_alert(Alert(AlertSeverity.MEDIUM, message, PROTOCOL)) - if float(breach) != previous_breach: - set_cache_value(CACHE_KEY_FLOOR_BREACH, float(breach)) + set_fresh_cache_value(CACHE_KEY_FLOOR_BREACH, float(breach)) def check_protocol_paused(is_paused: bool) -> None: @@ -888,7 +906,7 @@ def check_protocol_paused(is_paused: bool) -> None: """ logger.info("Protocol IS_PAUSED: %s", is_paused) - previous_paused = get_cache_value(CACHE_KEY_IS_PAUSED) + previous_paused = get_fresh_cache_value(CACHE_KEY_IS_PAUSED) if is_paused and previous_paused == 0: message = ( f"🚨 *3Jane Protocol PAUSED*\n" @@ -896,8 +914,7 @@ def check_protocol_paused(is_paused: bool) -> None: f"🔗 [ProtocolConfig](https://etherscan.io/address/{PROTOCOL_CONFIG_ADDRESS})" ) send_alert(Alert(AlertSeverity.CRITICAL, message, PROTOCOL)) - if float(is_paused) != previous_paused: - set_cache_value(CACHE_KEY_IS_PAUSED, float(is_paused)) + set_fresh_cache_value(CACHE_KEY_IS_PAUSED, float(is_paused)) def main() -> None: diff --git a/protocols/cap/README.md b/protocols/cap/README.md index 3af1fe1..4ddc4e7 100644 --- a/protocols/cap/README.md +++ b/protocols/cap/README.md @@ -29,3 +29,6 @@ Large mint monitoring is integrated into [liquidity.py](./liquidity.py). It intentionally does **not** scan events. Instead, it compares cached `totalSupply` values and alerts when the increase is above: - `CUSD_LARGE_MINT_THRESHOLD_PERCENT` (default: `0.05`, i.e. `5%` of previous `totalSupply`) + +The daily supply baseline expires after 36 hours and initializes from the next valid observation, preventing multiple +missed daily runs from being treated as one large mint interval. diff --git a/protocols/cap/liquidity.py b/protocols/cap/liquidity.py index 12fd3ee..58a77a3 100644 --- a/protocols/cap/liquidity.py +++ b/protocols/cap/liquidity.py @@ -2,7 +2,12 @@ from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + DAILY_CACHE_STALE_AFTER_SECONDS, + cache_filename, + get_fresh_last_value_for_key_from_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.config import Config from utils.logger import get_logger @@ -77,7 +82,9 @@ def main(): # --- cUSD Large Mint Monitoring (No Event Scanning) --- current_supply_raw = int(ctoken.functions.totalSupply().call()) - last_supply_cached = _to_int(get_last_value_for_key_from_file(cache_filename, CACHE_KEY_LAST_SUPPLY)) + last_supply_cached = _to_int( + get_fresh_last_value_for_key_from_file(cache_filename, CACHE_KEY_LAST_SUPPLY, DAILY_CACHE_STALE_AFTER_SECONDS) + ) if last_supply_cached > 0: delta_raw = current_supply_raw - last_supply_cached threshold_raw = int(last_supply_cached * MINT_THRESHOLD_PERCENT) @@ -99,7 +106,7 @@ def main(): ) send_alert(Alert(AlertSeverity.LOW, msg, PROTOCOL)) - write_last_value_to_file(cache_filename, CACHE_KEY_LAST_SUPPLY, current_supply_raw) + write_last_value_with_timestamp_to_file(cache_filename, CACHE_KEY_LAST_SUPPLY, current_supply_raw) if __name__ == "__main__": diff --git a/protocols/infinifi/README.md b/protocols/infinifi/README.md index 778c176..11b9d95 100644 --- a/protocols/infinifi/README.md +++ b/protocols/infinifi/README.md @@ -28,6 +28,11 @@ It compares cached `totalSupply` deltas and alerts when the increase is above: - `IUSD_LARGE_MINT_THRESHOLD_PERCENT` (default: `0.05`, i.e. `5%` of previous `totalSupply`) +## Cache Freshness + +Hourly delta baselines expire after 3 hours and initialize from the next valid observation. Breach-dedupe state and +liquid-reserve crossing detection are re-armed after the same monitoring gap. PPS-style loss baselines are not affected. + ### Emergency dispatch HIGH and CRITICAL alerts automatically trigger a signed webhook to diff --git a/protocols/infinifi/main.py b/protocols/infinifi/main.py index fb338c8..ef44bd5 100644 --- a/protocols/infinifi/main.py +++ b/protocols/infinifi/main.py @@ -5,7 +5,14 @@ from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + HOURLY_CACHE_STALE_AFTER_SECONDS, + cache_filename, + cache_key_is_stale, + get_fresh_last_value_for_key_from_file, + get_last_value_for_key_from_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.config import Config from utils.logger import get_logger @@ -74,16 +81,16 @@ def to_float(value, default=0.0): def send_breach_alert_once(cache_key, alert_message, severity=AlertSeverity.HIGH): - last_state = int(get_last_value_for_key_from_file(cache_filename, cache_key)) + last_state = int( + get_fresh_last_value_for_key_from_file(cache_filename, cache_key, HOURLY_CACHE_STALE_AFTER_SECONDS) + ) if last_state == 0: send_alert(Alert(severity, alert_message, PROTOCOL)) - write_last_value_to_file(cache_filename, cache_key, 1) + write_last_value_with_timestamp_to_file(cache_filename, cache_key, 1) def clear_breach_state(cache_key): - last_state = int(get_last_value_for_key_from_file(cache_filename, cache_key)) - if last_state == 1: - write_last_value_to_file(cache_filename, cache_key, 0) + write_last_value_with_timestamp_to_file(cache_filename, cache_key, 0) def _format_iusd_units(raw_value: int) -> Decimal: @@ -185,7 +192,11 @@ def main(): # Alert 0: Large iUSD mint by supply delta (no event scanning) cache_key_large_mints = f"{PROTOCOL}_large_mints_last_supply" - last_supply_cached = int(get_last_value_for_key_from_file(cache_filename, cache_key_large_mints)) + last_supply_cached = int( + get_fresh_last_value_for_key_from_file( + cache_filename, cache_key_large_mints, HOURLY_CACHE_STALE_AFTER_SECONDS + ) + ) if last_supply_cached > 0: delta_raw = iusd_supply_raw - last_supply_cached threshold_raw = int(last_supply_cached * MINT_THRESHOLD_PERCENT) @@ -211,13 +222,23 @@ def main(): PROTOCOL, ) ) - write_last_value_to_file(cache_filename, cache_key_large_mints, iusd_supply_raw) + write_last_value_with_timestamp_to_file(cache_filename, cache_key_large_mints, iusd_supply_raw) # Alert 1: Low Liquid Reserves if liquid_reserves > 0: cache_key_reserves = f"{PROTOCOL}_liquid_reserves" last_reserves = float(get_last_value_for_key_from_file(cache_filename, cache_key_reserves)) + if last_reserves > 0 and cache_key_is_stale( + cache_filename, cache_key_reserves, HOURLY_CACHE_STALE_AFTER_SECONDS + ): + logger.info( + "Liquid reserves cache timestamp is missing, invalid, or older than %sh; " + "re-arming crossing detection", + HOURLY_CACHE_STALE_AFTER_SECONDS // 3600, + ) + last_reserves = LIQUID_RESERVES_THRESHOLD + if ( last_reserves != 0 and liquid_reserves < LIQUID_RESERVES_THRESHOLD @@ -233,7 +254,7 @@ def main(): ) send_alert(Alert(AlertSeverity.HIGH, msg, PROTOCOL)) - write_last_value_to_file(cache_filename, cache_key_reserves, liquid_reserves) + write_last_value_with_timestamp_to_file(cache_filename, cache_key_reserves, liquid_reserves) # Alert 2 and Alert 3 intentionally disabled: # reserveRatio and illiquidTargetRatio have been persistently violated since inception, @@ -306,7 +327,14 @@ def main(): farm_label = farm.get("label", farm.get("name", farm_address)) cache_key_farm_ratio = f"{PROTOCOL}_farm_ratio_{farm_address.lower()}" - last_ratio = to_float(get_last_value_for_key_from_file(cache_filename, cache_key_farm_ratio)) + last_ratio_raw = get_last_value_for_key_from_file(cache_filename, cache_key_farm_ratio) + last_ratio = to_float(last_ratio_raw) + if last_ratio > 0 and cache_key_is_stale( + cache_filename, cache_key_farm_ratio, HOURLY_CACHE_STALE_AFTER_SECONDS + ): + write_last_value_with_timestamp_to_file(cache_filename, cache_key_farm_ratio, farm_ratio) + continue + if last_ratio > 0: ratio_change_pct = abs(farm_ratio - last_ratio) / last_ratio # skip farm if ratio change is less than 1% of TVL @@ -322,7 +350,6 @@ def main(): "change_pct": ratio_change_pct, } ) - write_last_value_to_file(cache_filename, cache_key_farm_ratio, farm_ratio) else: # Farm had no previous ratio (or previously zero). Alert if now materially active. if farm_ratio > FARM_RATIO_ACTIVATION_ALERT_THRESHOLD: @@ -332,7 +359,7 @@ def main(): "new_ratio": farm_ratio, } ) - write_last_value_to_file(cache_filename, cache_key_farm_ratio, farm_ratio) + write_last_value_with_timestamp_to_file(cache_filename, cache_key_farm_ratio, farm_ratio) if moved_farms: moved_farms.sort(key=lambda x: x["change_pct"], reverse=True) diff --git a/protocols/maple/README.md b/protocols/maple/README.md index 6362dc5..a8ab155 100644 --- a/protocols/maple/README.md +++ b/protocols/maple/README.md @@ -66,6 +66,10 @@ uv run maple/main.py Runs hourly via the [monitoring runner](../automation/jobs.yaml). +TVL delta baselines expire after 3 hours and initialize from the next valid observation. A zero delegate-cover state +is re-armed after the same monitoring gap. PPS and non-zero cover-decrease history remain persistent because cumulative +losses remain actionable. + ## Risk Report Full risk assessment: [maple-syrupusdc report](https://github.com/tapired/risk-score/blob/master/reports/report/maple-syrupusdc.md) diff --git a/protocols/maple/main.py b/protocols/maple/main.py index 8dfc244..74493a5 100644 --- a/protocols/maple/main.py +++ b/protocols/maple/main.py @@ -16,7 +16,15 @@ from protocols.maple.collateral import check_collateral_risk from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_path, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + HOURLY_CACHE_STALE_AFTER_SECONDS, + cache_key_is_stale, + cache_path, + get_fresh_last_value_for_key_from_file, + get_last_value_for_key_from_file, + write_last_value_to_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.formatting import format_usd from utils.logger import get_logger @@ -84,6 +92,20 @@ def set_cache_value(key: str, value: float) -> None: write_last_value_to_file(CACHE_FILENAME, key, value) +def get_fresh_cache_value(key: str) -> float: + """Read an hourly baseline, returning zero when it is stale.""" + val = get_fresh_last_value_for_key_from_file(CACHE_FILENAME, key, HOURLY_CACHE_STALE_AFTER_SECONDS) + try: + return float(val) + except (ValueError, TypeError): + return 0.0 + + +def set_fresh_cache_value(key: str, value: float) -> None: + """Write an hourly baseline and its observation timestamp.""" + write_last_value_with_timestamp_to_file(CACHE_FILENAME, key, value) + + def check_pps(client, pool) -> float: """Check Price Per Share and alert on decrease.""" pps = client.execute(pool.functions.convertToAssets(ONE_SHARE).call) @@ -113,7 +135,7 @@ def check_tvl(client, pool) -> float: total_assets = client.execute(pool.functions.totalAssets().call) tvl_usd = total_assets / ONE_SHARE - previous_tvl = get_cache_value(CACHE_KEY_TVL) + previous_tvl = get_fresh_cache_value(CACHE_KEY_TVL) logger.info("syrupUSDC TVL: %s (previous: %s)", format_usd(tvl_usd), format_usd(previous_tvl)) if previous_tvl > 0: @@ -128,8 +150,7 @@ def check_tvl(client, pool) -> float: ) send_alert(Alert(AlertSeverity.HIGH, message, PROTOCOL)) - if tvl_usd != previous_tvl: - set_cache_value(CACHE_KEY_TVL, tvl_usd) + set_fresh_cache_value(CACHE_KEY_TVL, tvl_usd) return tvl_usd @@ -290,14 +311,21 @@ def check_delegate_cover(client) -> None: cover_usd = cover_balance / ONE_SHARE previous_cover = get_cache_value(CACHE_KEY_DELEGATE_COVER) + cover_cache_is_stale = cache_key_is_stale( + CACHE_FILENAME, CACHE_KEY_DELEGATE_COVER, HOURLY_CACHE_STALE_AFTER_SECONDS + ) logger.info("Pool Delegate Cover: %s (previous: %s)", format_usd(cover_usd), format_usd(previous_cover)) if cover_usd == 0: - # Only alert once when cover is first detected as zero (previous > 0 or first run) - if previous_cover > 0: + if previous_cover > 0 or cover_cache_is_stale: + previous_line = ( + f"📊 Cover balance dropped from {format_usd(previous_cover)} to $0\n" + if previous_cover > 0 + else "📊 Cover balance is $0 after a stale or missing observation baseline\n" + ) message = ( f"🚨 *Maple syrupUSDC Pool Delegate Cover Empty*\n" - f"📊 Cover balance dropped from {format_usd(previous_cover)} to $0\n" + f"{previous_line}" f"⚠️ No delegate skin-in-the-game — reduced accountability for loan defaults\n" f"🔗 [PoolDelegateCover](https://etherscan.io/address/{POOL_DELEGATE_COVER})" ) @@ -311,8 +339,7 @@ def check_delegate_cover(client) -> None: ) send_alert(Alert(AlertSeverity.MEDIUM, message, PROTOCOL)) - if cover_usd != previous_cover: - set_cache_value(CACHE_KEY_DELEGATE_COVER, cover_usd) + set_fresh_cache_value(CACHE_KEY_DELEGATE_COVER, cover_usd) def main() -> None: diff --git a/protocols/strata/README.md b/protocols/strata/README.md index fa5ac2f..c242e19 100644 --- a/protocols/strata/README.md +++ b/protocols/strata/README.md @@ -38,3 +38,7 @@ Reference report: https://curation.yearn.fi/report/strata-srusde/ - Senior coverage ratio: daily - Strategy sUSDe balance: daily - Protocol TVL changes: daily + +Daily TVL, junior-assets, and strategy-ratio baselines expire after 36 hours and initialize from the next valid +observation. Senior-coverage dedupe is re-armed after the same monitoring gap. Share-rate monotonicity and cooldown +history remain persistent. diff --git a/protocols/strata/main.py b/protocols/strata/main.py index 8275fb1..6513d82 100644 --- a/protocols/strata/main.py +++ b/protocols/strata/main.py @@ -1,7 +1,14 @@ from web3 import Web3 from utils.abi import load_abi -from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + DAILY_CACHE_STALE_AFTER_SECONDS, + cache_filename, + get_fresh_last_value_for_key_from_file, + get_last_value_for_key_from_file, + write_last_value_to_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.logger import get_logger from utils.telegram import send_error_message, send_telegram_message @@ -60,16 +67,30 @@ def _set_cache_float(key: str, value: float) -> None: write_last_value_to_file(cache_filename, key, value) +def _fresh_cache_float(key: str) -> float | None: + value = get_fresh_last_value_for_key_from_file(cache_filename, key, DAILY_CACHE_STALE_AFTER_SECONDS) + if value == 0: + return None + try: + return float(value) + except ValueError: + return None + + +def _set_fresh_cache_float(key: str, value: float) -> None: + write_last_value_with_timestamp_to_file(cache_filename, key, value) + + def _breach_once(cache_key: str, condition: bool, message: str, messages: list[str]) -> None: - raw_state = get_last_value_for_key_from_file(cache_filename, cache_key) + raw_state = get_fresh_last_value_for_key_from_file(cache_filename, cache_key, DAILY_CACHE_STALE_AFTER_SECONDS) state = int(float(raw_state)) if raw_state != 0 else 0 if condition: if state == 0: messages.append(message) - write_last_value_to_file(cache_filename, cache_key, 1) - elif state == 1: - write_last_value_to_file(cache_filename, cache_key, 0) + write_last_value_with_timestamp_to_file(cache_filename, cache_key, 1) + else: + write_last_value_with_timestamp_to_file(cache_filename, cache_key, 0) def _check_susde_vault(messages: list[str], client, susde_vault, cooldown_contract) -> None: @@ -111,7 +132,7 @@ def _check_susde_vault(messages: list[str], client, susde_vault, cooldown_contra def _check_daily_tvl(messages: list[str], total_deposits: float) -> None: tvl_cache_key = f"{PROTOCOL}_total_deposits" - previous_total_deposits = _cache_float(tvl_cache_key) + previous_total_deposits = _fresh_cache_float(tvl_cache_key) if previous_total_deposits is not None and previous_total_deposits > 0: tvl_change = (total_deposits - previous_total_deposits) / previous_total_deposits if abs(tvl_change) >= TVL_CHANGE_ALERT_RATIO: @@ -119,12 +140,12 @@ def _check_daily_tvl(messages: list[str], total_deposits: float) -> None: "⚠️ Strata total TVL changed significantly.\n" f"previous: ${previous_total_deposits:,.2f} current: ${total_deposits:,.2f} ({tvl_change:.2%})" ) - _set_cache_float(tvl_cache_key, total_deposits) + _set_fresh_cache_float(tvl_cache_key, total_deposits) def _check_jr_drain(messages: list[str], jr_assets: float) -> None: jr_assets_cache_key = f"{PROTOCOL}_jr_assets" - previous_jr_assets = _cache_float(jr_assets_cache_key) + previous_jr_assets = _fresh_cache_float(jr_assets_cache_key) if previous_jr_assets is not None and previous_jr_assets > 0: jr_change = (jr_assets - previous_jr_assets) / previous_jr_assets if jr_change <= -JR_DRAIN_ALERT_RATIO: @@ -132,7 +153,7 @@ def _check_jr_drain(messages: list[str], jr_assets: float) -> None: "⚠️ jrUSDe totalAssets dropped quickly (junior side draining).\n" f"previous: ${previous_jr_assets:,.2f} current: ${jr_assets:,.2f} ({jr_change:.2%})" ) - _set_cache_float(jr_assets_cache_key, jr_assets) + _set_fresh_cache_float(jr_assets_cache_key, jr_assets) def main() -> None: @@ -201,7 +222,7 @@ def main() -> None: _set_cache_float(sr_rate_cache_key, sr_rate) strategy_ratio_cache_key = f"{PROTOCOL}_strategy_ratio" - previous_strategy_ratio = _cache_float(strategy_ratio_cache_key) + previous_strategy_ratio = _fresh_cache_float(strategy_ratio_cache_key) if previous_strategy_ratio is not None and previous_strategy_ratio > 0: strategy_ratio_drop = (previous_strategy_ratio - strategy_ratio) / previous_strategy_ratio if strategy_ratio_drop >= STRATEGY_RATIO_DROP_ALERT: @@ -210,7 +231,7 @@ def main() -> None: f"previous ratio: {previous_strategy_ratio:.2%} current ratio: {strategy_ratio:.2%} " f"({strategy_ratio_drop:.2%} drop)" ) - _set_cache_float(strategy_ratio_cache_key, strategy_ratio) + _set_fresh_cache_float(strategy_ratio_cache_key, strategy_ratio) _check_daily_tvl(messages, total_deposits) _check_jr_drain(messages, jr_assets) _check_susde_vault(messages, client, susde_vault, susde_cooldown) diff --git a/protocols/usdai/README.md b/protocols/usdai/README.md index a26f86c..ed98542 100644 --- a/protocols/usdai/README.md +++ b/protocols/usdai/README.md @@ -45,6 +45,12 @@ It runs cached `totalSupply` delta checks and alerts when the increase is above: The monitoring runner runs this monitor hourly (see `automation/jobs.yaml`). +## Cache Freshness + +The large-mint baseline expires after 3 hours and initializes from the next valid observation. Backing-invariant +dedupe is re-armed after the same monitoring gap. Loan-principal history remains persistent so activity during an +outage is still reported. + ## Price Monitoring Scope `usdai/main.py` does not monitor PYUSD/USD price. diff --git a/protocols/usdai/main.py b/protocols/usdai/main.py index 6744c32..e18843e 100644 --- a/protocols/usdai/main.py +++ b/protocols/usdai/main.py @@ -4,7 +4,14 @@ from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + HOURLY_CACHE_STALE_AFTER_SECONDS, + cache_filename, + get_fresh_last_value_for_key_from_file, + get_last_value_for_key_from_file, + write_last_value_to_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.config import Config from utils.logger import get_logger @@ -37,16 +44,16 @@ def send_breach_alert_once(cache_key, alert_message, severity=AlertSeverity.HIGH): - last_state = int(get_last_value_for_key_from_file(cache_filename, cache_key)) + last_state = int( + get_fresh_last_value_for_key_from_file(cache_filename, cache_key, HOURLY_CACHE_STALE_AFTER_SECONDS) + ) if last_state == 0: send_alert(Alert(severity, alert_message, PROTOCOL)) - write_last_value_to_file(cache_filename, cache_key, 1) + write_last_value_with_timestamp_to_file(cache_filename, cache_key, 1) def clear_breach_state(cache_key): - last_state = int(get_last_value_for_key_from_file(cache_filename, cache_key)) - if last_state == 1: - write_last_value_to_file(cache_filename, cache_key, 0) + write_last_value_with_timestamp_to_file(cache_filename, cache_key, 0) def get_loan_details(client, owner_addr): diff --git a/protocols/ustb/README.md b/protocols/ustb/README.md index 4b78ce0..96b7d9f 100644 --- a/protocols/ustb/README.md +++ b/protocols/ustb/README.md @@ -29,6 +29,9 @@ Monitors the [Superstate Short Duration US Government Securities Fund (USTB)](ht - **Large supply change**: HIGH alert if USTB total supply changes by more than **+/-10%** compared to the previous hourly run. +The supply baseline expires after 3 hours and initializes from the next valid observation. NAV monotonicity history +does not expire because a cumulative NAV decrease remains actionable after an outage. + ### Oracle Staleness - **Stale oracle**: HIGH alert if the latest checkpoint `effectiveAt` timestamp is more than **4 days** (345,600s) old. The SuperstateOracle reverts at 5 days (432,000s), so this provides a 1-day buffer. diff --git a/protocols/ustb/main.py b/protocols/ustb/main.py index c957970..2448ae9 100644 --- a/protocols/ustb/main.py +++ b/protocols/ustb/main.py @@ -14,7 +14,14 @@ from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_path, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + HOURLY_CACHE_STALE_AFTER_SECONDS, + cache_path, + get_fresh_last_value_for_key_from_file, + get_last_value_for_key_from_file, + write_last_value_to_file, + write_last_value_with_timestamp_to_file, +) from utils.chainlink import FeedReading, read_feeds, scale_price from utils.chains import Chain from utils.formatting import format_usd @@ -201,11 +208,13 @@ def _check_supply_change(total_supply_raw: int, nav_price: float) -> None: """Alert if total supply changed by more than 10% since the previous hourly run.""" total_supply = _to_tokens(total_supply_raw) - prev_supply_str = str(get_last_value_for_key_from_file(CACHE_FILE, CACHE_KEY_SUPPLY)) + prev_supply_str = str( + get_fresh_last_value_for_key_from_file(CACHE_FILE, CACHE_KEY_SUPPLY, HOURLY_CACHE_STALE_AFTER_SECONDS) + ) if prev_supply_str == "0": logger.info("No cached supply found, initialising cache") - write_last_value_to_file(CACHE_FILE, CACHE_KEY_SUPPLY, total_supply_raw) + write_last_value_with_timestamp_to_file(CACHE_FILE, CACHE_KEY_SUPPLY, total_supply_raw) return prev_supply_raw = int(prev_supply_str) @@ -226,7 +235,7 @@ def _check_supply_change(total_supply_raw: int, nav_price: float) -> None: ) ) - write_last_value_to_file(CACHE_FILE, CACHE_KEY_SUPPLY, total_supply_raw) + write_last_value_with_timestamp_to_file(CACHE_FILE, CACHE_KEY_SUPPLY, total_supply_raw) def _check_oracle_staleness(current_timestamp: int, effective_at: int) -> None: diff --git a/tests/test_3jane.py b/tests/test_3jane.py index facd498..cf1f14e 100644 --- a/tests/test_3jane.py +++ b/tests/test_3jane.py @@ -22,6 +22,13 @@ def stub_cache(monkeypatch: pytest.MonkeyPatch, module: ModuleType) -> dict[str, cache: dict[str, str] = {} monkeypatch.setattr(module, "get_last_value_for_key_from_file", lambda _filename, key: cache.get(key, 0)) monkeypatch.setattr(module, "set_cache_value", lambda key, value: cache.__setitem__(key, str(value))) + monkeypatch.setattr(module, "get_fresh_cache_value", lambda key: float(cache.get(key, 0))) + monkeypatch.setattr(module, "set_fresh_cache_value", lambda key, value: cache.__setitem__(key, str(value))) + monkeypatch.setattr( + module, + "get_fresh_last_value_for_key_from_file", + lambda _filename, key, _stale_after: cache.get(key, 0), + ) return cache diff --git a/tests/test_cache_sqlite.py b/tests/test_cache_sqlite.py index 01dc4e6..d856e2d 100644 --- a/tests/test_cache_sqlite.py +++ b/tests/test_cache_sqlite.py @@ -1,7 +1,13 @@ from __future__ import annotations +from pathlib import Path + +import pytest + from utils import cache, paths, store +NOW = 2_000_000_000 + def _use_cache_dir(monkeypatch, tmp_path) -> None: monkeypatch.setattr(paths, "CACHE_DIR", str(tmp_path)) @@ -74,3 +80,41 @@ def test_dual_write_legacy(monkeypatch, tmp_path): assert store.state_get("cache-id.txt", "aave") == "7" assert filename.read_text() == "aave:7\n" + + +def test_timestamped_cache_value_is_fresh_through_boundary( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _use_cache_dir(monkeypatch, tmp_path) + filename = str(tmp_path / "cache-id.txt") + max_age = 10 + cache.write_last_value_with_timestamp_to_file(filename, "reserves", 42, NOW - max_age) + + assert cache.get_fresh_last_value_for_key_from_file(filename, "reserves", max_age, NOW) == "42" + + +def test_timestamped_cache_value_is_absent_when_stale( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _use_cache_dir(monkeypatch, tmp_path) + filename = str(tmp_path / "cache-id.txt") + cache.write_last_value_with_timestamp_to_file(filename, "reserves", 42, NOW - 11) + + assert cache.get_fresh_last_value_for_key_from_file(filename, "reserves", 10, NOW) == 0 + + +def test_existing_cache_with_missing_or_invalid_timestamp_is_stale( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + _use_cache_dir(monkeypatch, tmp_path) + filename = str(tmp_path / "cache-id.txt") + cache.write_last_value_to_file(filename, "reserves", 42) + + assert cache.cache_key_is_stale(filename, "reserves", 10, NOW) + + for timestamp in (0, -1, "bad", float("nan"), float("inf"), NOW + 1): + cache.write_last_value_to_file(filename, cache.cache_timestamp_key("reserves"), timestamp) + assert cache.cache_key_is_stale(filename, "reserves", 10, NOW) diff --git a/utils/cache.py b/utils/cache.py index 8e42ae0..b17e4f3 100644 --- a/utils/cache.py +++ b/utils/cache.py @@ -1,4 +1,5 @@ import os +import time from typing import Union from dotenv import load_dotenv @@ -20,6 +21,9 @@ # dedupe and morpho rows; the daily profile overrides MORPHO_FILENAME to isolate. morpho_filename: str = cache_path(os.getenv("MORPHO_FILENAME", "cache-id.txt")) +HOURLY_CACHE_STALE_AFTER_SECONDS = 3 * 60 * 60 +DAILY_CACHE_STALE_AFTER_SECONDS = 36 * 60 * 60 + def get_last_queued_id_from_file(protocol: str) -> int: return int(get_last_value_for_key_from_file(cache_filename, protocol)) @@ -76,6 +80,51 @@ def write_last_value_to_file(filename: str, write_key: str, write_value: Union[i _write_last_value_to_legacy_file(filename, write_key, write_value) +def cache_timestamp_key(value_key: str) -> str: + return f"{value_key}_ts" + + +def cache_key_is_stale( + filename: str, + value_key: str, + stale_after_seconds: int, + current_timestamp: int | None = None, +) -> bool: + if current_timestamp is None: + current_timestamp = int(time.time()) + + timestamp_raw = get_last_value_for_key_from_file(filename, cache_timestamp_key(value_key)) + try: + timestamp = int(timestamp_raw) + except (TypeError, ValueError, OverflowError): + return True + + return timestamp <= 0 or timestamp > current_timestamp or current_timestamp - timestamp > stale_after_seconds + + +def get_fresh_last_value_for_key_from_file( + filename: str, + wanted_key: str, + stale_after_seconds: int, + current_timestamp: int | None = None, +) -> Union[str, int]: + if cache_key_is_stale(filename, wanted_key, stale_after_seconds, current_timestamp): + return 0 + return get_last_value_for_key_from_file(filename, wanted_key) + + +def write_last_value_with_timestamp_to_file( + filename: str, + write_key: str, + write_value: Union[int, str, float], + current_timestamp: int | None = None, +) -> None: + if current_timestamp is None: + current_timestamp = int(time.time()) + write_last_value_to_file(filename, write_key, write_value) + write_last_value_to_file(filename, cache_timestamp_key(write_key), current_timestamp) + + def _get_last_value_from_legacy_file(filename: str, wanted_key: str) -> Union[str, int]: if not os.path.exists(filename): return 0 diff --git a/utils/erc20_supply_delta_monitor.py b/utils/erc20_supply_delta_monitor.py index b2c1bc0..984e910 100644 --- a/utils/erc20_supply_delta_monitor.py +++ b/utils/erc20_supply_delta_monitor.py @@ -12,7 +12,12 @@ from utils.abi import load_abi from utils.alert import Alert, AlertSeverity, send_alert -from utils.cache import cache_filename, get_last_value_for_key_from_file, write_last_value_to_file +from utils.cache import ( + HOURLY_CACHE_STALE_AFTER_SECONDS, + cache_filename, + get_fresh_last_value_for_key_from_file, + write_last_value_with_timestamp_to_file, +) from utils.chains import Chain from utils.logger import get_logger from utils.web3_wrapper import ChainManager @@ -87,7 +92,10 @@ def run_erc20_supply_delta_monitor(config: ERC20SupplyDeltaMonitorConfig) -> Non decimals, token_symbol = client.execute_batch(batch) current_supply_raw = int(token.functions.totalSupply().call()) - last_supply_cached = _to_int(get_last_value_for_key_from_file(cache_filename, _cache_key_last_supply(config))) + cache_key = _cache_key_last_supply(config) + last_supply_cached = _to_int( + get_fresh_last_value_for_key_from_file(cache_filename, cache_key, HOURLY_CACHE_STALE_AFTER_SECONDS) + ) if last_supply_cached > 0: delta_raw = current_supply_raw - last_supply_cached @@ -108,7 +116,7 @@ def run_erc20_supply_delta_monitor(config: ERC20SupplyDeltaMonitorConfig) -> Non ) ) - write_last_value_to_file(cache_filename, _cache_key_last_supply(config), current_supply_raw) + write_last_value_with_timestamp_to_file(cache_filename, cache_key, current_supply_raw) except Exception as exc: logger.error("ERC20 supply-delta monitor failed for %s: %s", config.protocol, exc)