WIP: Implement fallback oracles for perps

CHANGELOG.md

@@ -4,7 +4,33 @@ Update this for each program release and mainnet deployment.
 
 ## not on mainnet
 
-### v0.23.0, 2024-3-
+### v0.24.0, 2024-4-
+
+- Allow skipping banks and invalid oracles when computing health (#891)
+
+  This is only possible when we know for sure that the operation would not put the account into negative health zone.   
+
+- Add support for Raydium CLMM as oracle fallback (#856)
+
+- Add a `TokenBalanceLog` when charging collateral fees (#894)
+
+- Withdraw instruction: remove overflow error and return appropriate error message instead (#910) 
+
+- Banks: add more safety checks (#895)
+
+- Add a health check instruction (#913)
+
+  Assert in a transaction that operation run on a mango account does not reduce it's health below a specified amount.
+
+- Add a sequence check instruction (#909)
+
+  Assert that a transaction was emitted and run with a correct view of the current mango state.  
+
+## mainnet
+
+### v0.23.0, 2024-3-8
+
+Deployment: Mar 8, 2024 at 12:10:52 Central European Standard Time, https://explorer.solana.com/tx/6MXGookZoYGMYb7tWrrmgZzVA13HJimHNqwHRVFeqL9YpQD7YasH1pQn4MSQTK1o13ixKTGFxwZsviUzmHzzP9m
 
 - Allow disabling asset liquidations for tokens (#867)
 
@@ -26,8 +52,6 @@ Update this for each program release and mainnet deployment.
 - Flash loan: Add a "swap without flash loan fees" option (#882)
 - Cleanup, tests and minor (#878, #875, #854, #838, #895)
 
-## mainnet
-
 ### v0.22.0, 2024-3-3
 
 Deployment: Mar 3, 2024 at 23:52:08 Central European Standard Time, https://explorer.solana.com/tx/3MpEMU12Pv7RpSnwfShoM9sbyr41KAEeJFCVx9ypkq8nuK8Q5vm7CRLkdhH3u91yQ4k44a32armZHaoYguX6NqsY

bin/cli/src/main.rs

@@ -92,6 +92,31 @@ struct JupiterSwap {
     rpc: Rpc,
 }
 
+#[derive(Args, Debug, Clone)]
+struct SanctumSwap {
+    #[clap(long)]
+    account: String,
+
+    /// also pays for everything
+    #[clap(short, long)]
+    owner: String,
+
+    #[clap(long)]
+    input_mint: String,
+
+    #[clap(long)]
+    output_mint: String,
+
+    #[clap(short, long)]
+    amount: u64,
+
+    #[clap(short, long, default_value = "50")]
+    max_slippage_bps: u64,
+
+    #[clap(flatten)]
+    rpc: Rpc,
+}
+
 #[derive(ArgEnum, Clone, Debug)]
 #[repr(u8)]
 pub enum CliSide {
@@ -189,6 +214,7 @@ enum Command {
     CreateAccount(CreateAccount),
     Deposit(Deposit),
     JupiterSwap(JupiterSwap),
+    SanctumSwap(SanctumSwap),
     GroupAddress {
         #[clap(short, long)]
         creator: String,
@@ -312,6 +338,19 @@ async fn main() -> Result<(), anyhow::Error> {
                 .await?;
             println!("{}", txsig);
         }
+        Command::SanctumSwap(cmd) => {
+            let client = cmd.rpc.client(Some(&cmd.owner))?;
+            let account = pubkey_from_cli(&cmd.account);
+            let owner = Arc::new(keypair_from_cli(&cmd.owner));
+            let input_mint = pubkey_from_cli(&cmd.input_mint);
+            let output_mint = pubkey_from_cli(&cmd.output_mint);
+            let client = MangoClient::new_for_existing_account(client, account, owner).await?;
+            let txsig = client
+                .sanctum()
+                .swap(input_mint, output_mint, cmd.max_slippage_bps, cmd.amount)
+                .await?;
+            println!("{}", txsig);
+        }
         Command::GroupAddress { creator, num } => {
             let creator = pubkey_from_cli(&creator);
             println!("{}", MangoClient::group_for_admin(creator, num));

bin/cli/src/save_snapshot.rs

@@ -67,7 +67,7 @@ pub async fn save_snapshot(
     .await?;
 
     // Getting solana account snapshots via jsonrpc
-    snapshot_source::start(
+    let snapshot_job = snapshot_source::start(
         snapshot_source::Config {
             rpc_http_url: rpc_url.clone(),
             mango_group,
@@ -79,6 +79,11 @@ pub async fn save_snapshot(
         extra_accounts,
         account_update_sender,
     );
+    tokio::spawn(async move {
+        let res = snapshot_job.await;
+        tracing::error!("Snapshot job exited, terminating process.. ({:?})", res);
+        std::process::exit(-1);
+    });
 
     let mut chain_data = chain_data::ChainData::new();
 

bin/liquidator/Cargo.toml

@@ -42,11 +42,14 @@ shellexpand = "2.1.0"
 solana-account-decoder = { workspace = true }
 solana-client = { workspace = true }
 solana-logger = { workspace = true }
+solana-address-lookup-table-program = "~1.16.7"
 solana-rpc = { workspace = true }
 solana-sdk = { workspace = true }
 tokio = { version = "1", features = ["full"] }
 tokio-stream = { version = "0.1.9"}
 tokio-tungstenite = "0.16.1"
 tracing = "0.1"
 regex = "1.9.5"
-hdrhistogram = "7.5.4"
+hdrhistogram = "7.5.4"
+indexmap = "2.0.0"
+borsh = { version = "0.10.3", features = ["const-generics"] }

bin/liquidator/src/cli_args.rs

@@ -1,7 +1,7 @@
 use crate::trigger_tcs;
 use anchor_lang::prelude::Pubkey;
 use clap::Parser;
-use mango_v4_client::{jupiter, priority_fees_cli};
+use mango_v4_client::{priority_fees_cli, swap};
 use std::collections::HashSet;
 
 #[derive(Parser, Debug)]
@@ -28,11 +28,11 @@ pub(crate) enum JupiterVersionArg {
     V6,
 }
 
-impl From<JupiterVersionArg> for jupiter::Version {
+impl From<JupiterVersionArg> for swap::Version {
     fn from(a: JupiterVersionArg) -> Self {
         match a {
-            JupiterVersionArg::Mock => jupiter::Version::Mock,
-            JupiterVersionArg::V6 => jupiter::Version::V6,
+            JupiterVersionArg::Mock => swap::Version::Mock,
+            JupiterVersionArg::V6 => swap::Version::V6,
         }
     }
 }
@@ -121,6 +121,12 @@ pub struct Cli {
     #[clap(long, env, value_parser, value_delimiter = ',')]
     pub(crate) rebalance_alternate_jupiter_route_tokens: Option<Vec<u16>>,
 
+    /// query sanctum for routes to and from these tokens
+    ///
+    /// These routes will only be used when trying to rebalance a LST token
+    #[clap(long, env, value_parser, value_delimiter = ',')]
+    pub(crate) rebalance_alternate_sanctum_route_tokens: Option<Vec<u16>>,
+
     /// When closing borrows, the rebalancer can't close token positions exactly.
     /// Instead it purchases too much and then gets rid of the excess in a second step.
     /// If this is 0.05, then it'll swap borrow_value * (1 + 0.05) quote token into borrow token.
@@ -130,12 +136,25 @@ pub struct Cli {
     #[clap(long, env, default_value = "30")]
     pub(crate) rebalance_refresh_timeout_secs: u64,
 
+    #[clap(long, env, value_enum, default_value = "false")]
+    pub(crate) rebalance_using_limit_order: BoolArg,
+
+    /// distance (in bps) from oracle price at which to place order for rebalancing
+    #[clap(long, env, default_value = "100")]
+    pub(crate) rebalance_limit_order_distance_from_oracle_price_bps: u64,
+
     /// if taking tcs orders is enabled
     ///
     /// typically only disabled for tests where swaps are unavailable
     #[clap(long, env, value_enum, default_value = "true")]
     pub(crate) take_tcs: BoolArg,
 
+    #[clap(long, env, default_value = "30")]
+    pub(crate) tcs_refresh_timeout_secs: u64,
+
+    #[clap(long, env, default_value = "1000")]
+    pub(crate) tcs_check_interval_ms: u64,
+
     /// profit margin at which to take tcs orders
     #[clap(long, env, default_value = "0.0005")]
     pub(crate) tcs_profit_fraction: f64,
@@ -178,6 +197,10 @@ pub struct Cli {
     #[clap(long, env, default_value = "https://quote-api.jup.ag/v6")]
     pub(crate) jupiter_v6_url: String,
 
+    /// override the jupiter http request timeout
+    #[clap(long, env, default_value = "30")]
+    pub(crate) jupiter_timeout_secs: u64,
+
     /// provide a jupiter token, currently only for jup v6
     #[clap(long, env, default_value = "")]
     pub(crate) jupiter_token: String,
@@ -191,6 +214,12 @@ pub struct Cli {
     #[clap(long, env, value_enum, default_value = "true")]
     pub(crate) telemetry: BoolArg,
 
+    /// if liquidation is enabled
+    ///
+    /// might be used to run an instance of liquidator dedicated to TCS and another one for liquidation
+    #[clap(long, env, value_enum, default_value = "true")]
+    pub(crate) liquidation_enabled: BoolArg,
+
     /// liquidation refresh timeout in secs
     #[clap(long, env, default_value = "30")]
     pub(crate) liquidation_refresh_timeout_secs: u8,
@@ -216,4 +245,20 @@ pub struct Cli {
     /// how long should it wait before logging an oracle error again (for the same token)
     #[clap(long, env, default_value = "30")]
     pub(crate) skip_oracle_error_in_logs_duration_secs: u64,
+
+    /// max number of liquidation/tcs to do concurrently
+    #[clap(long, env, default_value = "5")]
+    pub(crate) max_parallel_operations: u64,
+
+    /// Also use sanctum for rebalancing
+    #[clap(long, env, value_enum, default_value = "false")]
+    pub(crate) sanctum_enabled: BoolArg,
+
+    /// override the url to sanctum
+    #[clap(long, env, default_value = "https://api.sanctum.so/v1")]
+    pub(crate) sanctum_url: String,
+
+    /// override the sanctum http request timeout
+    #[clap(long, env, default_value = "30")]
+    pub(crate) sanctum_timeout_secs: u64,
 }

bin/liquidator/src/liquidate.rs

@@ -9,6 +9,7 @@ use mango_v4_client::{chain_data, MangoClient, PreparedInstructions};
 use solana_sdk::signature::Signature;
 
 use futures::{stream, StreamExt, TryStreamExt};
+use mango_v4::accounts_ix::HealthCheckKind::MaintRatio;
 use rand::seq::SliceRandom;
 use tracing::*;
 use {anyhow::Context, fixed::types::I80F48, solana_sdk::pubkey::Pubkey};
@@ -260,7 +261,22 @@ impl<'a> LiquidateHelper<'a> {
             )
             .await
             .context("creating perp_liq_base_or_positive_pnl_instruction")?;
+
         liq_ixs.cu = liq_ixs.cu.max(self.config.compute_limit_for_liq_ix);
+
+        let liqor = &self.client.mango_account().await?;
+        liq_ixs.append(
+            self.client
+                .health_check_instruction(
+                    liqor,
+                    self.config.min_health_ratio,
+                    vec![],
+                    vec![*perp_market_index],
+                    MaintRatio,
+                )
+                .await?,
+        );
+
         let txsig = self
             .client
             .send_and_confirm_owner_tx(liq_ixs.to_instructions())
@@ -501,6 +517,20 @@ impl<'a> LiquidateHelper<'a> {
             .await
             .context("creating liq_token_with_token ix")?;
         liq_ixs.cu = liq_ixs.cu.max(self.config.compute_limit_for_liq_ix);
+
+        let liqor = self.client.mango_account().await?;
+        liq_ixs.append(
+            self.client
+                .health_check_instruction(
+                    &liqor,
+                    self.config.min_health_ratio,
+                    vec![asset_token_index, liab_token_index],
+                    vec![],
+                    MaintRatio,
+                )
+                .await?,
+        );
+
         let txsig = self
             .client
             .send_and_confirm_owner_tx(liq_ixs.to_instructions())
@@ -651,14 +681,11 @@ impl<'a> LiquidateHelper<'a> {
 }
 
 #[allow(clippy::too_many_arguments)]
-pub async fn maybe_liquidate_account(
+pub async fn can_liquidate_account(
     mango_client: &MangoClient,
     account_fetcher: &chain_data::AccountFetcher,
     pubkey: &Pubkey,
-    config: &Config,
 ) -> anyhow::Result<bool> {
-    let liqor_min_health_ratio = I80F48::from_num(config.min_health_ratio);
-
     let account = account_fetcher.fetch_mango_account(pubkey)?;
     let health_cache = mango_client
         .health_cache(&account)
@@ -675,6 +702,18 @@ pub async fn maybe_liquidate_account(
         "possible candidate",
     );
 
+    Ok(true)
+}
+
+#[allow(clippy::too_many_arguments)]
+pub async fn maybe_liquidate_account(
+    mango_client: &MangoClient,
+    account_fetcher: &chain_data::AccountFetcher,
+    pubkey: &Pubkey,
+    config: &Config,
+) -> anyhow::Result<bool> {
+    let liqor_min_health_ratio = I80F48::from_num(config.min_health_ratio);
+
     // Fetch a fresh account and re-compute
     // This is -- unfortunately -- needed because the websocket streams seem to not
     // be great at providing timely updates to the account data.