From 2d19a3e5e01cdbb70d7cd7cef28e27a6daa23778 Mon Sep 17 00:00:00 2001 From: zong-zhe Date: Thu, 19 May 2022 17:17:23 +0800 Subject: [PATCH] refactor: adjusted some macros in kclvm-parser and add some methods in kclvm-ast. Note: 1. Adjusted the position of some macros in kclvm-parser. What: 1. Move some util macros for ast from kclvm-parser to kclvm-ast, and add some new util methods in kclvm-ast. How: 1. Move macros "node_ref", "expr_as" and "stmt_as" form from kclvm-parser/mod.rs to kclvm-ast/lib.rs 2. Add method "build_assign_node" in kclvm-ast/lib.rs in order to construct an AssignStmt 3. Add method "filter_schema_stmt_from_module" in kclvm-ast/ast.rs in order to select all "SchemaStmt" in "Module". 4. Add method "TryFrom<...>" in kclvm-ast/ast.rs for "StringLit" and "NameConstant" in order to convert from rust string and bool to kclvm ast. Why: 1. Adjust the structure of the kclvm-ast and add some util methods to continuously enrich the infrastructure of KCLVM to support the development. --- kclvm/ast/src/ast.rs | 40 +++++++++- kclvm/ast/src/lib.rs | 49 ++++++++++++ kclvm/ast/src/tests.rs | 129 +++++++++++++++++++++++++++++++- kclvm/config/Cargo.lock | 4 +- kclvm/parser/Cargo.lock | 4 +- kclvm/parser/src/parser/expr.rs | 2 +- kclvm/parser/src/parser/mod.rs | 32 -------- kclvm/parser/src/parser/stmt.rs | 5 +- kclvm/parser/src/parser/ty.rs | 4 +- kclvm/runner/Cargo.lock | 4 +- kclvm/runtime/Cargo.lock | 8 +- kclvm/sema/Cargo.lock | 4 +- kclvm/src/lib.rs | 3 +- 13 files changed, 233 insertions(+), 55 deletions(-) diff --git a/kclvm/ast/src/ast.rs b/kclvm/ast/src/ast.rs index 5e14a40cc..69baa718c 100644 --- a/kclvm/ast/src/ast.rs +++ b/kclvm/ast/src/ast.rs @@ -42,7 +42,7 @@ use kclvm_span::Loc; use rustc_span::Pos; use super::token; - +use crate::node_ref; /// Node is the file, line and column number information /// that all AST nodes need to contain. #[derive(Serialize, Deserialize, Debug, Clone)] @@ -225,6 +225,19 @@ pub struct Module { pub comments: Vec>, } +impl Module { + /// Get all ast.schema_stmts from ast.module and return it in a Vec. + pub fn filter_schema_stmt_from_module(&self) -> Vec> { + let mut stmts = Vec::new(); + for stmt in &self.body { + if let Stmt::Schema(schema_stmt) = &stmt.node { + stmts.push(node_ref!(schema_stmt.clone())); + } + } + return stmts; + } +} + /* * Stmt */ @@ -998,6 +1011,19 @@ pub struct StringLit { pub value: String, } +/// Generate ast.StringLit from String +impl TryFrom for StringLit { + type Error = &'static str; + + fn try_from(value: String) -> Result { + Ok(Self { + value: value.clone(), + raw_value: format!("{:?}", value), + is_long_string: false, + }) + } +} + /// NameConstant, e.g. /// ```kcl /// True @@ -1024,6 +1050,18 @@ impl NameConstant { } } +/// Generate ast.NameConstant from Bool +impl TryFrom for NameConstant { + type Error = &'static str; + + fn try_from(value: bool) -> Result { + match value { + true => Ok(NameConstant::True), + false => Ok(NameConstant::False), + } + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct NameConstantLit { pub value: NameConstant, diff --git a/kclvm/ast/src/lib.rs b/kclvm/ast/src/lib.rs index 5ade77563..d0cc4279e 100644 --- a/kclvm/ast/src/lib.rs +++ b/kclvm/ast/src/lib.rs @@ -1,4 +1,5 @@ // Copyright 2021 The KCL Authors. All rights reserved. +use crate::ast::*; pub mod ast; pub mod token; @@ -9,3 +10,51 @@ pub mod walker; mod tests; pub const MAIN_PKG: &str = "__main__"; + +#[macro_export] +macro_rules! node_ref { + ($node: expr) => { + NodeRef::new(Node::dummy_node($node)) + }; + ($node: expr, $pos:expr) => { + NodeRef::new(Node::node_with_pos($node, $pos)) + }; +} + +#[macro_export] +macro_rules! expr_as { + ($expr: expr, $expr_enum: path) => { + if let $expr_enum(x) = ($expr.node as Expr) { + Some(x) + } else { + None + } + }; +} + +#[macro_export] +macro_rules! stmt_as { + ($stmt: expr, $stmt_enum: path) => { + if let $stmt_enum(x) = ($stmt.node as Stmt) { + Some(x) + } else { + None + } + }; +} + +/// Construct an AssignStmt node with assign_value as value +pub fn build_assign_node(attr_name: &str, assign_value: NodeRef) -> NodeRef { + let iden = node_ref!(Identifier { + names: vec![attr_name.to_string()], + pkgpath: String::new(), + ctx: ExprContext::Store + }); + + node_ref!(Stmt::Assign(AssignStmt { + value: assign_value, + targets: vec![iden], + type_annotation: None, + ty: None + })) +} diff --git a/kclvm/ast/src/tests.rs b/kclvm/ast/src/tests.rs index 57981695a..43acdc8bb 100644 --- a/kclvm/ast/src/tests.rs +++ b/kclvm/ast/src/tests.rs @@ -1,5 +1,7 @@ -use crate::ast; use crate::walker::MutSelfMutWalker; +use crate::AugOp::Mod; +use crate::{ast, build_assign_node, node_ref, Identifier, Module, Node, NodeRef, SchemaStmt}; +use std::process::id; fn get_dummy_assign_ast() -> ast::Node { let filename = "main.k"; @@ -140,3 +142,128 @@ fn test_mut_walker() { VarMutSelfMutWalker {}.walk_assign_stmt(&mut assign_stmt.node); assert_eq!(assign_stmt.node.targets[0].node.names[0], "x") } + +#[test] +fn test_try_from_for_stringlit() { + let str_lit = ast::StringLit::try_from("test_str".to_string()).unwrap(); + let json_str = serde_json::to_string(&str_lit).unwrap(); + + let str_expected = + r#"{"is_long_string":false,"raw_value":"\"test_str\"","value":"test_str"}"# + .to_string(); + assert_eq!(str_expected, json_str); +} + +#[test] +fn test_try_from_for_nameconstant() { + let name_cons = ast::NameConstant::try_from(true).unwrap(); + let json_str = serde_json::to_string(&name_cons).unwrap(); + assert_eq!("\"True\"", json_str); + + let name_cons = ast::NameConstant::try_from(false).unwrap(); + let json_str = serde_json::to_string(&name_cons).unwrap(); + assert_eq!("\"False\"", json_str); +} + +#[test] +fn test_filter_schema_with_no_schema() { + let ast_mod = Module { + filename: "".to_string(), + pkg: "".to_string(), + doc: "".to_string(), + name: "".to_string(), + body: vec![], + comments: vec![], + }; + let schema_stmts = ast_mod.filter_schema_stmt_from_module(); + assert_eq!(schema_stmts.len(), 0); +} + +#[test] +fn test_filter_schema_with_one_schema() { + let mut ast_mod = Module { + filename: "".to_string(), + pkg: "".to_string(), + doc: "".to_string(), + name: "".to_string(), + body: vec![], + comments: vec![], + }; + let mut gen_schema_stmts = gen_schema_stmt(1); + ast_mod.body.append(&mut gen_schema_stmts); + let schema_stmts = ast_mod.filter_schema_stmt_from_module(); + assert_eq!(schema_stmts.len(), 1); + assert_eq!(schema_stmts[0].node.name.node, "schema_stmt_0".to_string()); +} + +#[test] +fn test_filter_schema_with_mult_schema() { + let mut ast_mod = Module { + filename: "".to_string(), + pkg: "".to_string(), + doc: "".to_string(), + name: "".to_string(), + body: vec![], + comments: vec![], + }; + let mut gen_schema_stmts = gen_schema_stmt(10); + ast_mod.body.append(&mut gen_schema_stmts); + let schema_stmts = ast_mod.filter_schema_stmt_from_module(); + assert_eq!(schema_stmts.len(), 10); + for i in 0..10 { + assert_eq!( + schema_stmts[i].node.name.node, + "schema_stmt_".to_string() + &i.to_string() + ) + } +} + +#[test] +fn test_build_assign_stmt() { + let test_expr = node_ref!(ast::Expr::Identifier(Identifier { + names: vec!["name1".to_string(), "name2".to_string()], + pkgpath: "test".to_string(), + ctx: ast::ExprContext::Load + })); + let assgin_stmt = build_assign_node("test_attr_name", test_expr); + + if let ast::Stmt::Assign(ref assign) = assgin_stmt.node { + if let ast::Expr::Identifier(ref iden) = &assign.value.node { + assert_eq!(iden.names.len(), 2); + assert_eq!(iden.names[0], "name1".to_string()); + assert_eq!(iden.names[1], "name2".to_string()); + assert_eq!(iden.pkgpath, "test".to_string()); + match iden.ctx { + ast::ExprContext::Load => {} + _ => { + assert!(false); + } + } + } else { + assert!(false); + } + } else { + assert!(false); + } +} + +fn gen_schema_stmt(count: i32) -> Vec> { + let mut schema_stmts = Vec::new(); + for c in 0..count { + schema_stmts.push(node_ref!(ast::Stmt::Schema(SchemaStmt { + doc: "".to_string(), + name: node_ref!("schema_stmt_".to_string() + &c.to_string()), + parent_name: None, + for_host_name: None, + is_mixin: false, + is_protocol: false, + args: None, + mixins: vec![], + body: vec![], + decorators: vec![], + checks: vec![], + index_signature: None + }))) + } + schema_stmts +} diff --git a/kclvm/config/Cargo.lock b/kclvm/config/Cargo.lock index 94e9cd237..d8754d979 100644 --- a/kclvm/config/Cargo.lock +++ b/kclvm/config/Cargo.lock @@ -243,9 +243,9 @@ dependencies = [ [[package]] name = "ron" -version = "0.6.6" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4" +checksum = "1b861ecaade43ac97886a512b360d01d66be9f41f3c61088b42cedf92e03d678" dependencies = [ "base64", "bitflags", diff --git a/kclvm/parser/Cargo.lock b/kclvm/parser/Cargo.lock index a51c6e185..2061b67b4 100644 --- a/kclvm/parser/Cargo.lock +++ b/kclvm/parser/Cargo.lock @@ -923,9 +923,9 @@ dependencies = [ [[package]] name = "ron" -version = "0.6.6" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4" +checksum = "1b861ecaade43ac97886a512b360d01d66be9f41f3c61088b42cedf92e03d678" dependencies = [ "base64", "bitflags", diff --git a/kclvm/parser/src/parser/expr.rs b/kclvm/parser/src/parser/expr.rs index 01ee984eb..527cff408 100644 --- a/kclvm/parser/src/parser/expr.rs +++ b/kclvm/parser/src/parser/expr.rs @@ -8,8 +8,8 @@ use super::int::bytes_to_int; use super::Parser; use either::{self, Either}; +use kclvm_ast::node_ref; -use crate::node_ref; use crate::parser::precedence::Precedence; use kclvm_ast::ast::*; use kclvm_ast::token; diff --git a/kclvm/parser/src/parser/mod.rs b/kclvm/parser/src/parser/mod.rs index c4517418d..f82d4bab7 100644 --- a/kclvm/parser/src/parser/mod.rs +++ b/kclvm/parser/src/parser/mod.rs @@ -30,38 +30,6 @@ use kclvm_ast::token::{CommentKind, Token, TokenKind}; use kclvm_ast::token_stream::{Cursor, TokenStream}; use kclvm_span::symbol::Symbol; -#[macro_export] -macro_rules! node_ref { - ($node: expr) => { - NodeRef::new(Node::dummy_node($node)) - }; - ($node: expr, $pos:expr) => { - NodeRef::new(Node::node_with_pos($node, $pos)) - }; -} - -#[macro_export] -macro_rules! expr_as { - ($expr: expr, $expr_enum: path) => { - if let $expr_enum(x) = ($expr.node as Expr) { - Some(x) - } else { - None - } - }; -} - -#[macro_export] -macro_rules! stmt_as { - ($stmt: expr, $stmt_enum: path) => { - if let $stmt_enum(x) = ($stmt.node as Stmt) { - Some(x) - } else { - None - } - }; -} - pub struct Parser<'a> { /// The current token. pub token: Token, diff --git a/kclvm/parser/src/parser/stmt.rs b/kclvm/parser/src/parser/stmt.rs index b10847295..8baaa3076 100644 --- a/kclvm/parser/src/parser/stmt.rs +++ b/kclvm/parser/src/parser/stmt.rs @@ -3,15 +3,12 @@ use core::panic; -use kclvm_ast::ast::*; +use kclvm_ast::{ast::*, node_ref, expr_as}; use kclvm_ast::token::{DelimToken, LitKind, Token, TokenKind}; use kclvm_span::symbol::kw; use super::Parser; -use crate::expr_as; -use crate::node_ref; - /// Parser implementation of statements, which consists of expressions and tokens. /// Parser uses `parse_exprlist` and `parse_expr` in [`kclvm_parser::parser::expr`] /// to get a expression node, and then concretize it into the specified expression node, diff --git a/kclvm/parser/src/parser/ty.rs b/kclvm/parser/src/parser/ty.rs index 3f67d8a9e..566687cd5 100644 --- a/kclvm/parser/src/parser/ty.rs +++ b/kclvm/parser/src/parser/ty.rs @@ -2,9 +2,7 @@ use super::Parser; -use crate::expr_as; - -use kclvm_ast::ast; +use kclvm_ast::{ast, expr_as}; use kclvm_ast::ast::{Expr, Node, NodeRef, Type}; use kclvm_ast::token; use kclvm_ast::token::{BinOpToken, DelimToken, TokenKind}; diff --git a/kclvm/runner/Cargo.lock b/kclvm/runner/Cargo.lock index 09fcdf0a5..a79ac7844 100644 --- a/kclvm/runner/Cargo.lock +++ b/kclvm/runner/Cargo.lock @@ -1027,9 +1027,9 @@ dependencies = [ [[package]] name = "ron" -version = "0.6.6" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4" +checksum = "1b861ecaade43ac97886a512b360d01d66be9f41f3c61088b42cedf92e03d678" dependencies = [ "base64", "bitflags", diff --git a/kclvm/runtime/Cargo.lock b/kclvm/runtime/Cargo.lock index 8fcacc574..87b740726 100644 --- a/kclvm/runtime/Cargo.lock +++ b/kclvm/runtime/Cargo.lock @@ -24,9 +24,9 @@ dependencies = [ [[package]] name = "autocfg" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" @@ -151,9 +151,9 @@ checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" [[package]] name = "indexmap" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" +checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", "hashbrown", diff --git a/kclvm/sema/Cargo.lock b/kclvm/sema/Cargo.lock index 950d1f97d..0ee37fe80 100644 --- a/kclvm/sema/Cargo.lock +++ b/kclvm/sema/Cargo.lock @@ -1082,9 +1082,9 @@ dependencies = [ [[package]] name = "ron" -version = "0.6.6" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86018df177b1beef6c7c8ef949969c4f7cb9a9344181b92486b23c79995bdaa4" +checksum = "1b861ecaade43ac97886a512b360d01d66be9f41f3c61088b42cedf92e03d678" dependencies = [ "base64", "bitflags", diff --git a/kclvm/src/lib.rs b/kclvm/src/lib.rs index 21ec384dc..7b510241d 100644 --- a/kclvm/src/lib.rs +++ b/kclvm/src/lib.rs @@ -146,7 +146,8 @@ pub extern "C" fn kclvm_cli_run(args: *const i8, plugin_agent: *const i8) -> *co std::fs::remove_file(&ll_path).unwrap(); } ll_path_lock.unlock().unwrap(); - tx.send(dylib_path).expect("channel will be there waiting for the pool"); + tx.send(dylib_path) + .expect("channel will be there waiting for the pool"); }); } let dylib_paths = rx.iter().take(prog_count).collect::>();