add near-state-indexer

Cargo.toml

@@ -12,9 +12,32 @@ members = [
     "configuration",
     "database",
     "epoch-indexer",
+    "near-state-indexer",
     "perf-testing",
     "readnode-primitives",
     "rpc-server",
     "state-indexer",
     "tx-indexer",
 ]
+
+[workspace.dependencies]
+
+configuration = { path = "configuration" }
+database = { path = "database" }
+readnode-primitives = { path = "readnode-primitives" }
+epoch-indexer = { path = "epoch-indexer" }
+
+near-indexer = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-client = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-o11y = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-indexer-primitives = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-primitives = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-chain-configs = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-crypto = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+
+near-lake-framework = { git = 'https://github.com/kobayurii/near-lake-framework-rs.git', rev = 'dc7f2c8da1f5f683b37c2d23da428e8fae1fd897' }
+near-jsonrpc-client = { git = 'https://github.com/kobayurii/near-jsonrpc-client-rs.git', rev = 'bcff5ec4b77c1c935e5caf8eab790bf83968f8c5'}
+
+near-jsonrpc-primitives = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-parameters = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }
+near-vm-runner = { git = 'https://github.com/near/nearcore.git', rev = '30cc5f576f0ba5b500131fad4ac0ccb78b061476' }

configuration/Cargo.toml

@@ -35,7 +35,7 @@ tracing-subscriber = { version = "0.3.15", features = [
 tracing-opentelemetry = { version = "0.19", optional = true }
 tracing-stackdriver = "0.7.2" # GCP logs
 
-near-lake-framework = "0.7.7"
+near-lake-framework.workspace = true
 
 [features]
 tracing-instrumentation = ["dep:opentelemetry-jaeger", "dep:tracing-opentelemetry"]

configuration/src/configs/general.rs

@@ -2,7 +2,9 @@ use std::str::FromStr;
 
 use serde_derive::Deserialize;
 
-use crate::configs::{deserialize_data_or_env, deserialize_optional_data_or_env};
+use crate::configs::{
+    deserialize_data_or_env, deserialize_optional_data_or_env, required_value_or_panic,
+};
 
 #[derive(Debug, Clone)]
 pub struct GeneralRpcServerConfig {
@@ -38,6 +40,14 @@ pub struct GeneralStateIndexerConfig {
     pub concurrency: usize,
 }
 
+#[derive(Debug, Clone)]
+pub struct GeneralNearStateIndexerConfig {
+    pub chain_id: ChainId,
+    pub indexer_id: String,
+    pub metrics_server_port: u16,
+    pub concurrency: usize,
+}
+
 #[derive(Debug, Clone)]
 pub struct GeneralEpochIndexerConfig {
     pub chain_id: ChainId,
@@ -50,8 +60,8 @@ pub struct GeneralEpochIndexerConfig {
 pub struct CommonGeneralConfig {
     #[serde(deserialize_with = "deserialize_data_or_env")]
     pub chain_id: ChainId,
-    #[serde(deserialize_with = "deserialize_data_or_env")]
-    pub near_rpc_url: String,
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub near_rpc_url: Option<String>,
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
     pub near_archival_rpc_url: Option<String>,
     #[serde(default)]
@@ -61,6 +71,8 @@ pub struct CommonGeneralConfig {
     #[serde(default)]
     pub state_indexer: CommonGeneralStateIndexerConfig,
     #[serde(default)]
+    pub near_state_indexer: CommonGeneralStateIndexerConfig,
+    #[serde(default)]
     pub epoch_indexer: CommonGeneralEpochIndexerConfig,
 }
 
@@ -70,6 +82,8 @@ pub enum ChainId {
     #[default]
     Mainnet,
     Testnet,
+    Betanet,
+    Localnet,
 }
 
 impl FromStr for ChainId {
@@ -79,6 +93,8 @@ impl FromStr for ChainId {
         match s {
             "mainnet" => Ok(ChainId::Mainnet),
             "testnet" => Ok(ChainId::Testnet),
+            "localnet" => Ok(ChainId::Localnet),
+            "betanet" => Ok(ChainId::Betanet),
             _ => Err(anyhow::anyhow!("Invalid chain id")),
         }
     }
@@ -210,6 +226,40 @@ impl Default for CommonGeneralStateIndexerConfig {
     }
 }
 
+#[derive(Deserialize, Debug, Clone)]
+pub struct CommonGeneralNearStateIndexerConfig {
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub indexer_id: Option<String>,
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub metrics_server_port: Option<u16>,
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub concurrency: Option<usize>,
+}
+
+impl CommonGeneralNearStateIndexerConfig {
+    pub fn default_indexer_id() -> String {
+        "near-state-indexer".to_string()
+    }
+
+    pub fn default_metrics_server_port() -> u16 {
+        8082
+    }
+
+    pub fn default_concurrency() -> usize {
+        1
+    }
+}
+
+impl Default for CommonGeneralNearStateIndexerConfig {
+    fn default() -> Self {
+        Self {
+            indexer_id: Some(Self::default_indexer_id()),
+            metrics_server_port: Some(Self::default_metrics_server_port()),
+            concurrency: Some(Self::default_concurrency()),
+        }
+    }
+}
+
 #[derive(Deserialize, Debug, Clone)]
 pub struct CommonGeneralEpochIndexerConfig {
     #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
@@ -234,7 +284,7 @@ impl From<CommonGeneralConfig> for GeneralRpcServerConfig {
     fn from(common_config: CommonGeneralConfig) -> Self {
         Self {
             chain_id: common_config.chain_id,
-            near_rpc_url: common_config.near_rpc_url,
+            near_rpc_url: required_value_or_panic("near_rpc_url", common_config.near_rpc_url),
             near_archival_rpc_url: common_config.near_archival_rpc_url,
             referer_header_value: common_config
                 .rpc_server
@@ -269,7 +319,7 @@ impl From<CommonGeneralConfig> for GeneralTxIndexerConfig {
     fn from(common_config: CommonGeneralConfig) -> Self {
         Self {
             chain_id: common_config.chain_id,
-            near_rpc_url: common_config.near_rpc_url,
+            near_rpc_url: required_value_or_panic("near_rpc_url", common_config.near_rpc_url),
             near_archival_rpc_url: common_config.near_archival_rpc_url,
             indexer_id: common_config
                 .tx_indexer
@@ -291,7 +341,7 @@ impl From<CommonGeneralConfig> for GeneralStateIndexerConfig {
     fn from(common_config: CommonGeneralConfig) -> Self {
         Self {
             chain_id: common_config.chain_id,
-            near_rpc_url: common_config.near_rpc_url,
+            near_rpc_url: required_value_or_panic("near_rpc_url", common_config.near_rpc_url),
             near_archival_rpc_url: common_config.near_archival_rpc_url,
             indexer_id: common_config
                 .state_indexer
@@ -309,11 +359,31 @@ impl From<CommonGeneralConfig> for GeneralStateIndexerConfig {
     }
 }
 
+impl From<CommonGeneralConfig> for GeneralNearStateIndexerConfig {
+    fn from(common_config: CommonGeneralConfig) -> Self {
+        Self {
+            chain_id: common_config.chain_id,
+            indexer_id: common_config
+                .near_state_indexer
+                .indexer_id
+                .unwrap_or_else(CommonGeneralStateIndexerConfig::default_indexer_id),
+            metrics_server_port: common_config
+                .near_state_indexer
+                .metrics_server_port
+                .unwrap_or_else(CommonGeneralStateIndexerConfig::default_metrics_server_port),
+            concurrency: common_config
+                .near_state_indexer
+                .concurrency
+                .unwrap_or_else(CommonGeneralStateIndexerConfig::default_concurrency),
+        }
+    }
+}
+
 impl From<CommonGeneralConfig> for GeneralEpochIndexerConfig {
     fn from(common_config: CommonGeneralConfig) -> Self {
         Self {
             chain_id: common_config.chain_id,
-            near_rpc_url: common_config.near_rpc_url,
+            near_rpc_url: required_value_or_panic("near_rpc_url", common_config.near_rpc_url),
             near_archival_rpc_url: common_config.near_archival_rpc_url,
             indexer_id: common_config
                 .epoch_indexer

configuration/src/configs/lake.rs

@@ -1,17 +1,13 @@
 use near_lake_framework::near_indexer_primitives::near_primitives;
 use serde_derive::Deserialize;
 
-use crate::configs::deserialize_data_or_env;
+use crate::configs::{deserialize_optional_data_or_env, required_value_or_panic};
 
-#[derive(Deserialize, Debug, Clone, Default)]
+#[derive(Debug, Clone)]
 pub struct LakeConfig {
-    #[serde(deserialize_with = "deserialize_data_or_env")]
     pub aws_access_key_id: String,
-    #[serde(deserialize_with = "deserialize_data_or_env")]
     pub aws_secret_access_key: String,
-    #[serde(deserialize_with = "deserialize_data_or_env")]
     pub aws_default_region: String,
-    #[serde(deserialize_with = "deserialize_data_or_env")]
     pub aws_bucket_name: String,
 }
 
@@ -53,3 +49,38 @@ impl LakeConfig {
         ))
     }
 }
+
+#[derive(Deserialize, Debug, Clone, Default)]
+pub struct CommonLakeConfig {
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub aws_access_key_id: Option<String>,
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub aws_secret_access_key: Option<String>,
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub aws_default_region: Option<String>,
+    #[serde(deserialize_with = "deserialize_optional_data_or_env", default)]
+    pub aws_bucket_name: Option<String>,
+}
+
+impl From<CommonLakeConfig> for LakeConfig {
+    fn from(common_config: CommonLakeConfig) -> Self {
+        Self {
+            aws_access_key_id: required_value_or_panic(
+                "aws_access_key_id",
+                common_config.aws_access_key_id,
+            ),
+            aws_secret_access_key: required_value_or_panic(
+                "aws_secret_access_key",
+                common_config.aws_secret_access_key,
+            ),
+            aws_default_region: required_value_or_panic(
+                "aws_default_region",
+                common_config.aws_default_region,
+            ),
+            aws_bucket_name: required_value_or_panic(
+                "aws_bucket_name",
+                common_config.aws_bucket_name,
+            ),
+        }
+    }
+}

configuration/src/configs/mod.rs

@@ -14,6 +14,14 @@ lazy_static::lazy_static! {
     static ref RE_NAME_ENV: regex::Regex = regex::Regex::new(r"\$\{(?<env_name>\w+)}").unwrap();
 }
 
+fn required_value_or_panic<T>(config_name: &str, value: Option<T>) -> T {
+    if let Some(value) = value {
+        value
+    } else {
+        panic!("Config `{}` is required!", config_name)
+    }
+}
+
 fn get_env_var<T>(env_var_name: &str) -> anyhow::Result<T>
 where
     T: FromStr,
@@ -70,7 +78,7 @@ pub struct CommonConfig {
     pub general: general::CommonGeneralConfig,
     #[serde(default)]
     pub rightsizing: rightsizing::CommonRightsizingConfig,
-    pub lake_config: lake::LakeConfig,
+    pub lake_config: lake::CommonLakeConfig,
     pub database: database::CommonDatabaseConfig,
 }
 
@@ -89,7 +97,7 @@ impl Config for RpcServerConfig {
     fn from_common_config(common_config: CommonConfig) -> Self {
         Self {
             general: common_config.general.into(),
-            lake_config: common_config.lake_config,
+            lake_config: common_config.lake_config.into(),
             database: database::DatabaseRpcServerConfig::from(common_config.database).into(),
         }
     }
@@ -117,7 +125,7 @@ impl Config for TxIndexerConfig {
         Self {
             general: common_config.general.into(),
             rightsizing: common_config.rightsizing.into(),
-            lake_config: common_config.lake_config,
+            lake_config: common_config.lake_config.into(),
             database: database::DatabaseTxIndexerConfig::from(common_config.database).into(),
         }
     }
@@ -142,7 +150,30 @@ impl Config for StateIndexerConfig {
         Self {
             general: common_config.general.into(),
             rightsizing: common_config.rightsizing.into(),
-            lake_config: common_config.lake_config,
+            lake_config: common_config.lake_config.into(),
+            database: database::DatabaseStateIndexerConfig::from(common_config.database).into(),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct NearStateIndexerConfig {
+    pub general: general::GeneralNearStateIndexerConfig,
+    pub rightsizing: rightsizing::RightsizingConfig,
+    pub database: database::DatabaseConfig,
+}
+
+impl NearStateIndexerConfig {
+    pub fn state_should_be_indexed(&self, state_change_value: &StateChangeValueView) -> bool {
+        self.rightsizing.state_should_be_indexed(state_change_value)
+    }
+}
+
+impl Config for NearStateIndexerConfig {
+    fn from_common_config(common_config: CommonConfig) -> Self {
+        Self {
+            general: common_config.general.into(),
+            rightsizing: common_config.rightsizing.into(),
             database: database::DatabaseStateIndexerConfig::from(common_config.database).into(),
         }
     }
@@ -159,7 +190,7 @@ impl Config for EpochIndexerConfig {
     fn from_common_config(common_config: CommonConfig) -> Self {
         Self {
             general: common_config.general.into(),
-            lake_config: common_config.lake_config,
+            lake_config: common_config.lake_config.into(),
             database: database::DatabaseStateIndexerConfig::from(common_config.database).into(),
         }
     }

configuration/src/lib.rs

@@ -7,7 +7,8 @@ mod configs;
 pub use crate::configs::database::DatabaseConfig;
 pub use crate::configs::general::ChainId;
 pub use crate::configs::{
-    EpochIndexerConfig, RpcServerConfig, StateIndexerConfig, TxIndexerConfig,
+    EpochIndexerConfig, NearStateIndexerConfig, RpcServerConfig, StateIndexerConfig,
+    TxIndexerConfig,
 };
 
 pub async fn read_configuration<T>() -> anyhow::Result<T>

database/Cargo.toml

@@ -34,13 +34,13 @@ tokio = { version = "1.36.0", features = [
 tracing = "0.1.34"
 uuid = { version = "1.7.0", optional = true }
 
-configuration = { path = "../configuration" }
-readnode-primitives = { path = "../readnode-primitives" }
+configuration.workspace = true
+readnode-primitives.workspace = true
 
-near-chain-configs = "0.20.0"
-near-primitives = "0.20.0"
-near-crypto = "0.20.0"
-near-indexer-primitives = "0.20.0"
+near-chain-configs.workspace = true
+near-primitives.workspace = true
+near-crypto.workspace = true
+near-indexer-primitives.workspace = true
 
 [features]
 default = ["scylla_db"]