refactor some code

Cargo.toml

@@ -10,32 +10,38 @@ repository = "https://github.com/nix-community/nix4vscode"
 homepage = "https://github.com/nix-community/nix4vscode"
 authors = ["Loongtao Zhang <loongtao.zhang@outlook.com>"]
 
-[dependencies]
-serde = { version = "1.0", features = ["derive", "alloc", "rc"] }
+[workspace.dependencies]
+futures = "0.3"
+semver = "1.0"
+tracing = "0.1"
+anyhow = "1.0"
+serde = { version = "1.0", features = ["derive", "rc"] }
 serde_json = "1.0"
+lazy-regex = "3.4"
+chrono = "0.4"
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
 toml = "0.8"
-tokio = { version = "1.42", features = ["full"] }
-tracing = "0.1"
-tracing-subscriber = { version = "0.3", features = ["env-filter", "std"] }
-semver = "1.0"
-bitflags = { version = "2.6", features = ["serde", "std"] }
-reqwest = { version = "0.12", default-features = false, features = [
-  "rustls-tls",
-  "gzip",
-  "json",
-] }
-minijinja = { version = "2.5", features = ["loader"] }
+
+tracing = { workspace = true }
+tokio = { version = "1.43", features = ["full"] }
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+futures = { workspace = true }
+
+semver = { workspace = true }
+minijinja = { version = "2.6", features = ["loader"] }
 clap = { version = "4.5", features = ["derive"] }
-futures = "0.3"
-anyhow = "1.0"
+anyhow = { workspace = true }
 nixpkgs-fmt = "1.3"
 redb = "2.4"
 thiserror = "2.0"
-lazy-regex = "3.4"
-chrono = "0.4"
-derive = { path = "./derive" }
+lazy-regex = { workspace = true }
+derive = { path = "./crates/derive" }
+code-api = { path = "./crates/code_api" }
 rust-embed = "8.5"
-rusqlite = { version = "0.32.1", features = ["bundled"] }
+rusqlite = { version = "0.32", features = ["bundled"] }
 itertools = "0.14.0"
 
 [package.metadata.deb]

crates/code_api/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "code-api"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+futures = { workspace = true }
+tracing = { workspace = true }
+anyhow = { workspace = true }
+serde = { workspace = true }
+bitflags = { version = "2.7", features = ["serde", "std"] }
+async-stream = "0.3"
+serde_json = { workspace = true }
+chrono = { workspace = true }
+lazy-regex = { workspace = true }
+reqwest = { version = "0.12", default-features = false, features = [
+  "rustls-tls",
+  "gzip",
+  "json",
+] }
+
+derive = { path = "../derive" }

crates/code_api/src/code.rs

@@ -0,0 +1,19 @@
+// https://github.com/microsoft/vscode/blob/d187d50a482ff80dcf74c35affb09dda1a7cd2fe/src/vs/platform/extensionManagement/common/extensionGalleryService.ts
+mod enums;
+mod extensions;
+mod flags;
+mod gallery_extension;
+mod http_client;
+mod query;
+mod request_body;
+mod version;
+
+pub use extensions::*;
+pub use gallery_extension::*;
+
+pub use enums::*;
+pub use flags::*;
+pub use http_client::*;
+pub use query::*;
+pub use request_body::*;
+pub use version::*;

src/code/gallery_extension/extension_version.rs → crates/code_api/src/code/gallery_extension/extension_version.rs

@@ -1,6 +1,7 @@
+use anyhow::anyhow;
 use derive::api;
 
-use crate::{code::PropertyType, error::Error};
+use crate::code::PropertyType;
 
 use super::*;
 
@@ -29,7 +30,7 @@ impl IRawGalleryExtensionVersion {
             .position(|item| item.key == PropertyType::Engine.to_string())
         {
             Some(idx) => Ok(self.properties[idx].value.clone()),
-            None => Err(Error::AttributeMissing("engine".into()).into()),
+            None => Err(anyhow!("Missing attribute: engine")),
         }
     }
 

src/code/http_client.rs → crates/code_api/src/code/http_client.rs

@@ -1,12 +1,12 @@
 use crate::{
-    code::{
-        self, IQueryState, IRawGalleryExtensionsResult, IRawGalleryQueryResult, TargetPlatform,
-    },
+    code::{self, IRawGalleryExtensionsResult, IRawGalleryQueryResult, TargetPlatform},
     config::Extension,
-    error::Error,
 };
 
 use super::Query;
+use anyhow::anyhow;
+use async_stream::try_stream;
+use futures::stream::Stream;
 use tracing::*;
 
 #[derive(Debug, Clone)]
@@ -20,43 +20,42 @@ impl HttpClient {
         Ok(Self { client })
     }
 
-    pub async fn get_extension_response(
+    pub fn get_extension_response(
         &self,
-        extensions: &[Extension],
-    ) -> anyhow::Result<code::IRawGalleryQueryResult> {
-        let mut results = IRawGalleryQueryResult::default();
-        let extension_count: u64 = extensions.len() as u64;
+        extensions: Vec<Extension>,
+    ) -> impl Stream<Item = anyhow::Result<code::IRawGalleryExtensionsResult>> + '_ {
         let mut page_number: u64 = 1;
-        loop {
-            let query = Query::new(extensions, page_number);
-            let body = serde_json::to_string(&query)?;
-            trace!("send request: {body}");
-            let mut response = self
-                .client
-                .post("https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery")
-                .header(
-                    "Accept",
-                    "Application/json; charset=utf-8; api-version=7.2-preview.1",
-                )
-                .header("Content-Type", "application/json")
-                .body(body.clone())
-                .send()
-                .await?
-                .json::<IRawGalleryQueryResult>()
-                .await?;
 
-            if response.results.is_empty() {
-                break;
-            }
+        try_stream! {
+            loop {
+                let query = Query::new(&extensions, page_number);
+                let body = serde_json::to_string(&query)?;
+                trace!("send request: {body}");
+                let response = self
+                    .client
+                    .post("https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery")
+                    .header(
+                        "Accept",
+                        "Application/json; charset=utf-8; api-version=7.2-preview.1",
+                    )
+                    .header("Content-Type", "application/json")
+                    .body(body.clone())
+                    .send()
+                    .await?
+                    .json::<IRawGalleryQueryResult>()
+                    .await?;
 
-            results.results.append(&mut response.results);
-            if page_number * IQueryState::DEFAULT_PAGE_SIZE >= extension_count {
-                break;
-            }
+                if response.results.is_empty() {
+                    break;
+                }
 
-            page_number += 1;
+                for item in response.results {
+                    yield item
+                }
+
+                page_number += 1;
+            }
         }
-        Ok(results)
     }
 
     async fn inner_get_extension_target_platform(
@@ -79,7 +78,7 @@ impl HttpClient {
             .await?
             .json::<code::IRawGalleryQueryResult>()
             .await?;
-        txt.results.into_iter().next().ok_or(Error::Unknown.into())
+        txt.results.into_iter().next().ok_or(anyhow!("Unknown"))
     }
 
     pub async fn get_extension_target_platform(

src/jinja/system.rs → crates/code_api/src/config.rs

@@ -1,5 +1,13 @@
 use serde::{Deserialize, Serialize};
 
+#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
+pub struct Extension {
+    pub publisher_name: String,
+    pub extension_name: String,
+    pub asset_url: Option<String>,
+    pub system: Option<SystemContext>,
+}
+
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
 pub struct SystemContext {
     arch: String,

crates/code_api/src/lib.rs

@@ -0,0 +1,2 @@
+pub mod code;
+pub mod config;

src/code.rs

@@ -1,29 +1,19 @@
-// https://github.com/microsoft/vscode/blob/d187d50a482ff80dcf74c35affb09dda1a7cd2fe/src/vs/platform/extensionManagement/common/extensionGalleryService.ts
-mod enums;
-mod extensions;
-mod flags;
-mod gallery_extension;
-mod http_client;
-mod query;
-mod request_body;
-mod version;
-
+use std::pin::pin;
 use std::str::FromStr;
 
-pub use extensions::*;
+use code_api::code::is_version_valid;
+use code_api::code::AssetType;
+use code_api::code::HttpClient;
+use code_api::code::IRawGalleryExtension;
+use code_api::code::TargetPlatform;
 use futures::future::join_all;
-pub use gallery_extension::*;
+use futures::stream;
+use futures::StreamExt;
 
-pub use enums::*;
-pub use flags::*;
-pub use http_client::*;
-pub use query::*;
-pub use request_body::*;
 use semver::Version;
 use tracing::debug;
 use tracing::error;
 use tracing::trace;
-pub use version::*;
 
 use crate::config::Config;
 use crate::jinja::AssetUrlContext;
@@ -45,31 +35,32 @@ impl CodeNix {
     }
 
     pub async fn get_extensions(&mut self, generator: Generator<'static>) -> Vec<NixContext> {
-        let obj = self
-            .client
-            .get_extension_response(&self.config.handled_extensions)
-            .await
-            .unwrap();
+        let mut obj = vec![];
+        {
+            let mut iter = self
+                .client
+                .get_extension_response(self.config.handled_extensions.clone())
+                .filter_map(|item| async move {
+                    match item {
+                        Ok(v) => Some(v),
+                        Err(_) => None,
+                    }
+                })
+                .flat_map(|item| stream::iter(item.extensions));
 
-        let futures: Vec<_> = obj
-            .results
-            .into_iter()
-            .flat_map(|item| item.extensions.into_iter())
-            .filter(|item| {
-                match self
+            let mut iter = pin!(iter);
+            while let Some(val) = iter.next().await {
+                if self
                     .config
-                    .contains(&item.publisher.publisher_name, &item.extension_name)
+                    .contains(&val.publisher.publisher_name, &val.extension_name)
                 {
-                    true => true,
-                    false => {
-                        debug!(
-                            "extensions be filtered {}.{}",
-                            item.publisher.publisher_name, item.extension_name
-                        );
-                        false
-                    }
+                    obj.push(val);
                 }
-            })
+            }
+        }
+
+        let futures: Vec<_> = obj
+            .into_iter()
             .map(|item| {
                 trace!("aa");
                 let generator = generator.clone();

src/config.rs

@@ -1,18 +1,11 @@
 use anyhow::anyhow;
+use code_api::config::{Extension, SystemContext};
 use lazy_regex::regex;
 use serde::{Deserialize, Serialize};
 use tokio::fs;
 use tracing::error;
 
-use crate::jinja::{Generator, SystemContext};
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
-pub struct Extension {
-    pub publisher_name: String,
-    pub extension_name: String,
-    pub asset_url: Option<String>,
-    pub system: Option<SystemContext>,
-}
+use crate::jinja::Generator;
 
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone)]
 #[serde(untagged)]
@@ -28,7 +21,8 @@ pub struct Config {
     pub autogen_warning: Option<String>,
     pub extensions: Vec<ExtensionType>,
     pub system: Option<SystemContext>,
-    pub handled_extensions: Vec<Extension>,
+    #[serde(skip)]
+    pub(crate) handled_extensions: Vec<Extension>,
 }
 
 impl Config {

src/jinja.rs

@@ -1,10 +1,8 @@
 mod context;
 mod filters;
-mod system;
 
 pub use context::*;
 use filters::*;
-pub use system::*;
 
 use minijinja::Environment;
 

src/jinja/context.rs

@@ -1,12 +1,13 @@
 use std::sync::Arc;
 
+use code_api::code::TargetPlatform;
 use serde::{Deserialize, Serialize};
 
 mod asset_url;
 
 pub use asset_url::*;
 
-use crate::{code::TargetPlatform, config::Config};
+use crate::config::Config;
 
 #[derive(Debug, Default, Deserialize, Serialize)]
 pub struct GeneratorContext {