Merge pull request #6 from KusionStack/dev/chai2010/bugfix

fix: add pos to comment node

kclvm/parser/src/parser/expr.rs

@@ -1017,15 +1017,16 @@ impl<'a> Parser<'a> {
         while let Some(mut x) = elif_list.pop() {
             x.node.orelse = if_item.orelse;
 
-            let mut t = Box::new(Node::dummy_node(Expr::ListIfItem(x.node)));
-
-            t.filename = x.filename;
-            t.line = x.line;
-            t.column = x.column;
-            t.end_line = x.end_line;
-            t.end_column = x.end_column;
+            let t = Node {
+                node: Expr::ListIfItem(x.node),
+                filename: x.filename,
+                line: x.line,
+                column: x.column,
+                end_line: x.end_line,
+                end_column: x.end_column,
+            };
 
-            if_item.orelse = Some(t);
+            if_item.orelse = Some(Box::new(t));
         }
 
         Box::new(Node::node(
@@ -1415,15 +1416,16 @@ impl<'a> Parser<'a> {
         while let Some(mut x) = elif_list.pop() {
             x.node.orelse = if_entry.node.orelse;
 
-            let mut t = Box::new(Node::dummy_node(Expr::ConfigIfEntry(x.node)));
-
-            t.filename = x.filename;
-            t.line = x.line;
-            t.column = x.column;
-            t.end_line = x.end_line;
-            t.end_column = x.end_column;
+            let t = Node {
+                node: Expr::ConfigIfEntry(x.node),
+                filename: x.filename,
+                line: x.line,
+                column: x.column,
+                end_line: x.end_line,
+                end_column: x.end_column,
+            };
 
-            if_entry.node.orelse = Some(t);
+            if_entry.node.orelse = Some(Box::new(t));
         }
         Box::new(Node::new(
             Expr::ConfigIfEntry(if_entry.node),
@@ -1455,12 +1457,22 @@ impl<'a> Parser<'a> {
 
         let token = self.token;
 
-        let mut body = ConfigIfEntryExpr {
-            if_cond: Box::new(Node::dummy_node(Expr::NameConstantLit(NameConstantLit {
-                value: NameConstant::None, // ignore
-            }))),
-            items: vec![],
-            orelse: None,
+        let mut body = {
+            let node = Node {
+                node: Expr::NameConstantLit(NameConstantLit {
+                    value: NameConstant::None, // ignore
+                }),
+                filename: "".to_string(),
+                line: 0,
+                column: 0,
+                end_line: 0,
+                end_column: 0,
+            };
+            ConfigIfEntryExpr {
+                if_cond: Box::new(node),
+                items: vec![],
+                orelse: None,
+            }
         };
 
         fn parse_body_item(

kclvm/parser/src/parser/mod.rs

@@ -77,7 +77,7 @@ pub struct Parser<'a> {
 
 impl<'a> Parser<'a> {
     pub fn new(sess: &'a ParseSession, stream: TokenStream) -> Self {
-        let (non_comment_tokens, comments) = Parser::split_token_stream(stream);
+        let (non_comment_tokens, comments) = Parser::split_token_stream(&sess, stream);
 
         let mut parser = Parser {
             token: Token::dummy(),
@@ -175,7 +175,10 @@ impl<'a> Parser<'a> {
 }
 
 impl<'a> Parser<'a> {
-    fn split_token_stream(stream: TokenStream) -> (Vec<Token>, Vec<NodeRef<Comment>>) {
+    fn split_token_stream(
+        sess: &'a ParseSession,
+        stream: TokenStream,
+    ) -> (Vec<Token>, Vec<NodeRef<Comment>>) {
         use rustc_span::BytePos;
 
         let mut comments = Vec::new();
@@ -207,13 +210,25 @@ impl<'a> Parser<'a> {
             // split comments
             if matches!(tok.kind, TokenKind::DocComment(_)) {
                 match tok.kind {
-                    TokenKind::DocComment(comment) => match comment {
+                    TokenKind::DocComment(comment_kind) => match comment_kind {
                         CommentKind::Line(x) => {
-                            comments.push(NodeRef::new(kclvm_ast::ast::Node::dummy_node(
-                                Comment {
+                            use rustc_span::Pos;
+                            let lo = sess.source_map.lookup_char_pos(tok.span.lo());
+                            let hi = sess.source_map.lookup_char_pos(tok.span.hi());
+                            let filename: String = format!("{}", lo.file.name.prefer_remapped());
+
+                            let node = kclvm_ast::ast::Node {
+                                node: Comment {
                                     text: x.as_str().to_string(),
                                 },
-                            )));
+                                filename: filename,
+                                line: lo.line as u64,
+                                column: lo.col.to_usize() as u64,
+                                end_line: hi.line as u64,
+                                end_column: hi.col.to_usize() as u64,
+                            };
+
+                            comments.push(NodeRef::new(node));
                         }
                     },
                     _ => (),

kclvm/parser/src/parser/stmt.rs

@@ -761,7 +761,8 @@ impl<'a> Parser<'_> {
             {
                 self.bump_token(TokenKind::Colon);
                 let typ = self.parse_type_annotation();
-                (Some(node_ref!(typ.node.to_string())), Some(typ))
+
+                (Some(node_ref!(typ.node.to_string(), typ.pos())), Some(typ))
             } else {
                 (None, None)
             };
@@ -953,7 +954,14 @@ impl<'a> Parser<'_> {
             checks: body_checks,
             index_signature: body_index_signature,
 
-            name: node_ref!("".to_string()),
+            name: Box::new(Node {
+                node: "".to_string(),
+                filename: "".to_string(),
+                line: 0,
+                column: 0,
+                end_line: 0,
+                end_column: 0,
+            }),
             parent_name: None,
             for_host_name: None,
             is_mixin: false,

kclvm/parser/src/parser/tests.rs

@@ -664,7 +664,7 @@ fn lambda_expr_2() {
     check_parsing_expr(
         r####"lambda x: int -> int {x}"####,
         expect![[r#"
-        Node { node: Lambda(LambdaExpr { args: Some(Node { node: Arguments { args: [Node { node: Identifier { names: ["x"], pkgpath: "", ctx: Load }, filename: "", line: 1, column: 7, end_line: 1, end_column: 8 }], defaults: [None], type_annotation_list: [Some(Node { node: "int", filename: "", line: 1, column: 1, end_line: 1, end_column: 1 })], ty_list: [Some(Node { node: Basic(Int), filename: "", line: 1, column: 10, end_line: 1, end_column: 13 })] }, filename: "", line: 1, column: 7, end_line: 1, end_column: 13 }), return_type_str: Some("int"), body: [Node { node: Expr(ExprStmt { exprs: [Node { node: Identifier(Identifier { names: ["x"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 22, end_line: 1, end_column: 23 }] }), filename: "", line: 1, column: 22, end_line: 1, end_column: 23 }], return_ty: Some(Node { node: Basic(Int), filename: "", line: 1, column: 17, end_line: 1, end_column: 20 }) }), filename: "", line: 1, column: 0, end_line: 1, end_column: 24 }
+        Node { node: Lambda(LambdaExpr { args: Some(Node { node: Arguments { args: [Node { node: Identifier { names: ["x"], pkgpath: "", ctx: Load }, filename: "", line: 1, column: 7, end_line: 1, end_column: 8 }], defaults: [None], type_annotation_list: [Some(Node { node: "int", filename: "", line: 1, column: 10, end_line: 1, end_column: 13 })], ty_list: [Some(Node { node: Basic(Int), filename: "", line: 1, column: 10, end_line: 1, end_column: 13 })] }, filename: "", line: 1, column: 7, end_line: 1, end_column: 13 }), return_type_str: Some("int"), body: [Node { node: Expr(ExprStmt { exprs: [Node { node: Identifier(Identifier { names: ["x"], pkgpath: "", ctx: Load }), filename: "", line: 1, column: 22, end_line: 1, end_column: 23 }] }), filename: "", line: 1, column: 22, end_line: 1, end_column: 23 }], return_ty: Some(Node { node: Basic(Int), filename: "", line: 1, column: 17, end_line: 1, end_column: 20 }) }), filename: "", line: 1, column: 0, end_line: 1, end_column: 24 }
         "#]],
     );
 }

kclvm/parser/src/tests.rs

@@ -58,4 +58,19 @@ data2 = {
         {"filename":"hello.k","pkg":"__main__","doc":"","name":"__main__","body":[{"node":{"Assign":{"targets":[{"node":{"names":["data2"],"pkgpath":"","ctx":"Store"},"filename":"hello.k","line":2,"column":0,"end_line":2,"end_column":5}],"value":{"node":{"Config":{"items":[{"node":{"key":null,"value":{"node":{"Config":{"items":[{"node":{"key":{"node":{"Identifier":{"names":["key"],"pkgpath":"","ctx":"Load"}},"filename":"hello.k","line":3,"column":7,"end_line":3,"end_column":10},"value":{"node":{"StringLit":{"is_long_string":false,"raw_value":"\"value1\"","value":"value1"}},"filename":"hello.k","line":3,"column":13,"end_line":3,"end_column":21},"operation":"Override","insert_index":-1},"filename":"hello.k","line":3,"column":7,"end_line":3,"end_column":21}]}},"filename":"hello.k","line":3,"column":6,"end_line":3,"end_column":22},"operation":"Union","insert_index":-1},"filename":"hello.k","line":3,"column":4,"end_line":3,"end_column":22},{"node":{"key":null,"value":{"node":{"ConfigIfEntry":{"if_cond":{"node":{"Compare":{"left":{"node":{"Identifier":{"names":["a"],"pkgpath":"","ctx":"Load"}},"filename":"hello.k","line":4,"column":7,"end_line":4,"end_column":8},"ops":["Eq"],"comparators":[{"node":{"NumberLit":{"binary_suffix":null,"value":{"Int":123}}},"filename":"hello.k","line":4,"column":12,"end_line":4,"end_column":15}]}},"filename":"hello.k","line":4,"column":7,"end_line":4,"end_column":15},"items":[{"node":{"key":null,"value":{"node":{"ConfigIfEntry":{"if_cond":{"node":{"Compare":{"left":{"node":{"Identifier":{"names":["b"],"pkgpath":"","ctx":"Load"}},"filename":"hello.k","line":4,"column":20,"end_line":4,"end_column":21},"ops":["Eq"],"comparators":[{"node":{"NumberLit":{"binary_suffix":null,"value":{"Int":456}}},"filename":"hello.k","line":4,"column":25,"end_line":4,"end_column":28}]}},"filename":"hello.k","line":4,"column":20,"end_line":4,"end_column":28},"items":[{"node":{"key":{"node":{"Identifier":{"names":["key"],"pkgpath":"","ctx":"Load"}},"filename":"hello.k","line":4,"column":30,"end_line":4,"end_column":33},"value":{"node":{"StringLit":{"is_long_string":false,"raw_value":"\"value2\"","value":"value2"}},"filename":"hello.k","line":4,"column":36,"end_line":4,"end_column":44},"operation":"Override","insert_index":-1},"filename":"hello.k","line":4,"column":36,"end_line":4,"end_column":44}],"orelse":null}},"filename":"hello.k","line":4,"column":30,"end_line":5,"end_column":0},"operation":"Override","insert_index":-1},"filename":"hello.k","line":4,"column":30,"end_line":5,"end_column":0}],"orelse":null}},"filename":"hello.k","line":4,"column":17,"end_line":5,"end_column":0},"operation":"Union","insert_index":-1},"filename":"hello.k","line":4,"column":4,"end_line":5,"end_column":0}]}},"filename":"hello.k","line":2,"column":8,"end_line":5,"end_column":1},"type_annotation":null}},"filename":"hello.k","line":2,"column":0,"end_line":6,"end_column":0}],"comments":[]}
         "#]],
     );
+
+    check_parsing_file_ast_json(
+        "hello.k",
+        r####"
+# comment1
+a = 1
+# comment22
+b = 2
+# comment333
+c = 3 # comment4444
+        "####,
+        expect![[r###"
+        {"filename":"hello.k","pkg":"__main__","doc":"","name":"__main__","body":[{"node":{"Assign":{"targets":[{"node":{"names":["a"],"pkgpath":"","ctx":"Store"},"filename":"hello.k","line":3,"column":0,"end_line":3,"end_column":1}],"value":{"node":{"NumberLit":{"binary_suffix":null,"value":{"Int":1}}},"filename":"hello.k","line":3,"column":4,"end_line":3,"end_column":5},"type_annotation":null}},"filename":"hello.k","line":3,"column":0,"end_line":5,"end_column":0},{"node":{"Assign":{"targets":[{"node":{"names":["b"],"pkgpath":"","ctx":"Store"},"filename":"hello.k","line":5,"column":0,"end_line":5,"end_column":1}],"value":{"node":{"NumberLit":{"binary_suffix":null,"value":{"Int":2}}},"filename":"hello.k","line":5,"column":4,"end_line":5,"end_column":5},"type_annotation":null}},"filename":"hello.k","line":5,"column":0,"end_line":7,"end_column":0},{"node":{"Assign":{"targets":[{"node":{"names":["c"],"pkgpath":"","ctx":"Store"},"filename":"hello.k","line":7,"column":0,"end_line":7,"end_column":1}],"value":{"node":{"NumberLit":{"binary_suffix":null,"value":{"Int":3}}},"filename":"hello.k","line":7,"column":4,"end_line":7,"end_column":5},"type_annotation":null}},"filename":"hello.k","line":7,"column":0,"end_line":8,"end_column":0}],"comments":[{"node":{"text":"# comment1"},"filename":"hello.k","line":2,"column":0,"end_line":2,"end_column":10},{"node":{"text":"# comment22"},"filename":"hello.k","line":4,"column":0,"end_line":4,"end_column":11},{"node":{"text":"# comment333"},"filename":"hello.k","line":6,"column":0,"end_line":6,"end_column":12},{"node":{"text":"# comment4444"},"filename":"hello.k","line":7,"column":6,"end_line":7,"end_column":19}]}
+        "###]],
+    );
 }