refactor: Finance perf base tables (#18)

**Motivation:**

Change the logic behind all the financial performance base tables to
better capture price movement and other fixes

**Modifications:**

Describe the modifications you've done.

**Result:**

After your change, what will change.

aave_dbt/models/datamart/inv_defi_ref_rates_by_day.sql

@@ -0,0 +1,28 @@
+{{ config(materialized='table') }}
+
+with stables as (
+select 
+  block_day
+  , sum(earnings_usd) / sum(deposits_usd) * 365 as stable_reference_rate_ex_price
+  , sum(earnings_usd + price_change_usd) / sum(deposits_usd) * 365 as stable_reference_rate
+-- from dev.inv_defi_ref_stable_rate_base
+from {{ ref('inv_defi_ref_stable_rate_base') }}
+group by block_day
+)
+
+, eth as (
+select 
+  partition_date as block_day
+  , apr as eth_reference_rate
+-- from protocol_data_lake.beacon_chain_staking_returns_by_day
+from {{ source('protocol_data_lake', 'beacon_chain_staking_returns_by_day') }}
+)
+
+select 
+  s.block_day
+  , s.stable_reference_rate
+  , s.stable_reference_rate_ex_price
+  , last_value(e.eth_reference_rate ignore nulls) over (order by s.block_day range between unbounded preceding and current row) as eth_reference_rate
+from stables s 
+  left join eth e on s.block_day = e.block_day
+order by s.block_day

aave_dbt/models/datamart/inv_defi_ref_stable_rate_base.sql

@@ -1,168 +1,102 @@
 {{ config(materialized='table') }}
 
 with prices as (
-select 
+select
   block_day
-  , chain
-  , reserve
-  , usd_price
--- from warehouse.token_prices_by_day
-from {{ source('warehouse', 'token_prices_by_day') }}
+  , symbol
+  , price_usd as usd_price
+-- from protocol_data_lake.coingecko_data_by_day 
+from {{ source('protocol_data_lake', 'coingecko_data_by_day') }}
+where true
+  and chain = 'ethereum' 
+  and symbol in ('USDT','USDC','DAI')
+)
+, compound as (
+select
+  block_day as block_time
+  , compound_version as market
+  , underlying_symbol as symbol
+  , deposits as deposits_native
+  , supply_apy as deposit_apy
+-- from protocol_data_lake.compound_v2_by_day
+from {{ source('protocol_data_lake', 'compound_v2_by_day') }}
+union all 
+select
+  block_day as block_time
+  , compound_version as market
+  , underlying_symbol as symbol
+  , deposits as deposits_native
+  , supply_apy as deposit_apy
+-- from protocol_data_lake.compound_v3_by_day
+from {{ source('protocol_data_lake', 'compound_v3_by_day') }}
+union all 
+select
+  block_hour as block_time
+  , compound_version as market
+  , underlying_symbol as symbol
+  , deposits as deposits_native
+  , supply_apy as deposit_apy
+-- from protocol_data_lake.compound_v2_by_hour
+from {{ source('protocol_data_lake', 'compound_v2_by_hour') }}
+union all 
+select
+  block_hour as block_time
+  , compound_version as market
+  , underlying_symbol as symbol
+  , deposits as deposits_native
+  , supply_apy as deposit_apy
+-- from protocol_data_lake.compound_v3_by_hour
+from {{ source('protocol_data_lake', 'compound_v3_by_hour') }}
 )
 
-, times as (
-select distinct
-  block_time
--- from datamart.market_state_by_time
+, combined_daily as (
+select
+  date_trunc(block_time, day) as block_day
+  , 'aave_' || right(market, 2) as market
+  , reserve_symbol as symbol
+  , avg(atoken_supply) as deposits_native
+  , avg(deposit_apy) as deposit_apy
+-- from datamart.market_state_by_time 
 from {{ ref('market_state_by_time') }}
-where market in ('ethereum_v2','ethereum_v3')
-order by block_time
+where market in ('ethereum_v2', 'ethereum_v3') and reserve_symbol in ('USDT','USDC','DAI')
+group by date_trunc(block_time, day), market, reserve_symbol
+union all 
+select
+  date_trunc(block_time, day) as block_day
+  , market
+  , symbol
+  , avg(deposits_native) as deposits_native
+  , avg(deposit_apy) as deposit_apy
+from compound
+group by date_trunc(block_time, day), market, symbol
 )
 
-, all_markets as (
-select 
-  t.block_time
-  , last_value('aave_' || right(a.market, 2) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.reserve_symbol ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.stable_debt_usd + a.variable_debt_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value(a.deposits_usd - (a.stable_debt_usd + a.variable_debt_usd) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.deposit_apy ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join datamart.market_state_by_time a
-  left join {{ ref('market_state_by_time') }} a
-    on (t.block_time = a.block_time and a.market = 'ethereum_v2' and a.reserve_symbol = 'USDT')
-union all
-select 
-  t.block_time
-  , last_value('aave_' || right(a.market, 2) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.reserve_symbol ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.stable_debt_usd + a.variable_debt_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value(a.deposits_usd - (a.stable_debt_usd + a.variable_debt_usd) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.deposit_apy ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join datamart.market_state_by_time a
-  left join {{ ref('market_state_by_time') }} a
-    on (t.block_time = a.block_time and a.market = 'ethereum_v2' and a.reserve_symbol = 'USDC')
-union all
-select 
-  t.block_time
-  , last_value('aave_' || right(a.market, 2) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.reserve_symbol ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.stable_debt_usd + a.variable_debt_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value(a.deposits_usd - (a.stable_debt_usd + a.variable_debt_usd) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.deposit_apy ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join datamart.market_state_by_time a
-  left join {{ ref('market_state_by_time') }} a
-    on (t.block_time = a.block_time and a.market = 'ethereum_v2' and a.reserve_symbol = 'DAI')
-union all 
+, pre_calc as (
 select 
-  t.block_time
-  , last_value('aave_' || right(a.market, 2) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.reserve_symbol ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.stable_debt_usd + a.variable_debt_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value(a.deposits_usd - (a.stable_debt_usd + a.variable_debt_usd) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.deposit_apy ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join datamart.market_state_by_time a
-  left join {{ ref('market_state_by_time') }} a
-    on (t.block_time = a.block_time and a.market = 'ethereum_v3' and a.reserve_symbol = 'USDT')
-union all
-select 
-  t.block_time
-  , last_value('aave_' || right(a.market, 2) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.reserve_symbol ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.stable_debt_usd + a.variable_debt_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value(a.deposits_usd - (a.stable_debt_usd + a.variable_debt_usd) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.deposit_apy ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join datamart.market_state_by_time a
-  left join {{ ref('market_state_by_time') }} a
-    on (t.block_time = a.block_time and a.market = 'ethereum_v3' and a.reserve_symbol = 'USDC')
-union all
-select 
-  t.block_time
-  , last_value('aave_' || right(a.market, 2) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.reserve_symbol ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.stable_debt_usd + a.variable_debt_usd ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value(a.deposits_usd - (a.stable_debt_usd + a.variable_debt_usd) ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.deposit_apy ignore nulls) over (order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join datamart.market_state_by_time a
-  left join {{ ref('market_state_by_time') }} a
-    on (t.block_time = a.block_time and a.market = 'ethereum_v3' and a.reserve_symbol = 'DAI')
-union all
-select 
-  t.block_time
-  , last_value(a.compound_version ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.underlying_symbol ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.borrows * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value((a.deposits + a.borrows) * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.supply_apy ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join protocol_data_lake.compound_v2_by_hour a
-  left join {{ source('protocol_data_lake','compound_v2_by_hour') }} a
-    on (t.block_time = a.block_hour)
-  left join prices p on (date_trunc(a.block_hour, day) = p.block_day and a.underlying_address = p.reserve and a.chain = p.chain)
-union all 
-select 
-  t.block_time
-  , last_value(a.compound_version ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.underlying_symbol ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.borrows * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value((a.deposits + a.borrows) * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.supply_apy ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join protocol_data_lake.compound_v3_by_hour a
-  left join {{ source('protocol_data_lake','compound_v3_by_hour') }} a
-    on (t.block_time = a.block_hour)
-  left join prices p on (date_trunc(a.block_hour, day) = p.block_day and a.underlying_address = p.reserve and a.chain = p.chain)
-union all
-select 
-  t.block_time
-  , last_value(a.compound_version ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.underlying_symbol ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.borrows * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value((a.deposits + a.borrows) * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.supply_apy ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join protocol_data_lake.compound_v2_by_day a
-  left join {{ source('protocol_data_lake','compound_v2_by_day') }} a
-    on (t.block_time = a.block_day)
-  left join prices p on (date_trunc(a.block_day, day) = p.block_day and a.underlying_address = p.reserve and a.chain = p.chain)
-union all 
-select 
-  t.block_time
-  , last_value(a.compound_version ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as market
-  , last_value(a.underlying_symbol ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as symbol
-  , last_value(a.deposits * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposits_usd
-  , last_value(a.borrows * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as borrows_usd
-  , last_value((a.deposits + a.borrows) * p.usd_price ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as tvl_usd
-  , last_value(a.supply_apy ignore nulls) over (partition by symbol order by t.block_time range between unbounded preceding and current row) as deposit_apy
-from times t 
-  -- left join protocol_data_lake.compound_v3_by_day a
-  left join {{ source('protocol_data_lake','compound_v3_by_day') }} a
-    on (t.block_time = a.block_day)
-  left join prices p on (date_trunc(a.block_day, day) = p.block_day and a.underlying_address = p.reserve and a.chain = p.chain)
+  d.block_day
+  , d.market
+  , d.symbol 
+  , d.deposits_native
+  , d.deposit_apy
+  , p.usd_price as price_usd
+  , coalesce(lead(usd_price) over (partition by d.market, d.symbol order by d.block_day), usd_price) as end_price_usd
+  , d.deposits_native * p.usd_price as deposits_usd
+from combined_daily d
+left join prices p on (d.block_day = p.block_day and d.symbol = p.symbol)
+order by block_day, market, symbol
 )
+
 select 
-  block_time 
+  block_day
   , market 
-  , symbol 
-  , deposits_usd 
-  , borrows_usd 
-  , tvl_usd 
+  , symbol
+  , deposits_native 
   , deposit_apy 
-  , deposits_usd * deposit_apy as deposits_mul_apy
-from all_markets
-where deposits_usd > 0
-order by block_time, market, symbol
+  , price_usd 
+  , end_price_usd
+  , deposits_native * price_usd as deposits_usd
+  , deposits_native * deposit_apy / 365 as earnings_native
+  , deposits_native * deposit_apy / 365 * price_usd as earnings_usd
+  , deposits_native * (1 + deposit_apy / 365) * (end_price_usd - price_usd) as price_change_usd 
+from pre_calc
+order by block_day, market, symbol 

aave_dbt/models/datamart/inv_treasury_atoken_performance_base.sql

@@ -105,11 +105,14 @@ select
   , balance_eth 
   , usd_price
   , usd_price / eth_usd as eth_price
-  , end_balance_native * (end_usd_price - usd_price) as price_change_usd
-  , end_balance_native * (end_usd_price / end_eth_usd - usd_price / eth_usd) as price_change_eth
   , earnings_native
   , earnings_usd
   , earnings_eth
+  , (balance_native + earnings_native) * (end_usd_price - usd_price) as price_change_usd
+  , (balance_native + earnings_native) * (end_usd_price / end_eth_usd - usd_price / eth_usd) as price_change_eth
+  , end_balance_native - earnings_native - balance_native as receipts_native
+  , (end_balance_native - earnings_native - balance_native) * usd_price as receipts_usd
+  , (end_balance_native - earnings_native - balance_native) * usd_price / eth_usd as receipts_eth
 from calcs 
 where 1=1
   and balance_native > 0