[ES|QL] Use custom AST for suggestion, validation, etc...

packages/kbn-monaco/index.ts

@@ -12,7 +12,7 @@ export { monaco } from './src/monaco_imports';
 export { XJsonLang } from './src/xjson';
 export { SQLLang } from './src/sql';
 export { ESQL_LANG_ID, ESQL_THEME_ID, ESQLLang } from './src/esql';
-export type { ESQLCustomAutocompleteCallbacks } from './src/esql';
+export type { ESQLCallbacks } from './src/esql';
 
 export * from './src/painless';
 /* eslint-disable-next-line @kbn/eslint/module_migration */

packages/kbn-monaco/src/esql/antlr/esql_lexer.g4

@@ -4,26 +4,27 @@
  * 2.0; you may not use this file except in compliance with the Elastic License
  * 2.0.
  */
-
+ 
 lexer grammar esql_lexer;
+options {  }
 
 DISSECT : D I S S E C T -> pushMode(EXPRESSION);
-GROK : G R O K -> pushMode(EXPRESSION);
+DROP : D R O P -> pushMode(SOURCE_IDENTIFIERS);
+ENRICH : E N R I C H -> pushMode(SOURCE_IDENTIFIERS);
 EVAL : E V A L -> pushMode(EXPRESSION);
-EXPLAIN : E X P L A I N -> pushMode(EXPLAIN_MODE);
 FROM : F R O M -> pushMode(SOURCE_IDENTIFIERS);
+GROK : G R O K -> pushMode(EXPRESSION);
+KEEP : K E E P -> pushMode(SOURCE_IDENTIFIERS);
+LIMIT : L I M I T -> pushMode(EXPRESSION);
+MV_EXPAND : M V UNDERSCORE E X P A N D -> pushMode(SOURCE_IDENTIFIERS);
+PROJECT : P R O J E C T -> pushMode(SOURCE_IDENTIFIERS);
+RENAME : R E N A M E -> pushMode(SOURCE_IDENTIFIERS);
 ROW : R O W -> pushMode(EXPRESSION);
+SHOW : S H O W -> pushMode(EXPRESSION);
+SORT : S O R T -> pushMode(EXPRESSION);
 STATS : S T A T S -> pushMode(EXPRESSION);
 WHERE : W H E R E -> pushMode(EXPRESSION);
-SORT : S O R T -> pushMode(EXPRESSION);
-MV_EXPAND : M V UNDERSCORE E X P A N D -> pushMode(EXPRESSION);
-LIMIT : L I M I T -> pushMode(EXPRESSION);
-PROJECT : P R O J E C T -> pushMode(EXPRESSION);
-DROP : D R O P -> pushMode(EXPRESSION);
-RENAME : R E N A M E -> pushMode(EXPRESSION);
-SHOW : S H O W -> pushMode(EXPRESSION);
-ENRICH : E N R I C H -> pushMode(ENRICH_IDENTIFIERS);
-KEEP : K E E P -> pushMode(EXPRESSION);
+UNKNOWN_CMD : ~[ \r\n\t[\]/]+ -> pushMode(EXPRESSION);
 
 LINE_COMMENT
     : '//' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN)
@@ -36,12 +37,7 @@ MULTILINE_COMMENT
 WS
     : [ \r\n\t]+ -> channel(HIDDEN)
     ;
-mode EXPLAIN_MODE;
-EXPLAIN_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(DEFAULT_MODE);
-EXPLAIN_PIPE : '|' -> type(PIPE), popMode;
-EXPLAIN_WS : WS -> channel(HIDDEN);
-EXPLAIN_LINE_COMMENT : LINE_COMMENT -> channel(HIDDEN);
-EXPLAIN_MULTILINE_COMMENT : MULTILINE_COMMENT -> channel(HIDDEN);
+
 mode EXPRESSION;
 
 PIPE : '|' -> popMode;
@@ -82,169 +78,60 @@ DECIMAL_LITERAL
     | DOT DIGIT+ EXPONENT
     ;
 
-BY : 'by';
-
-DATE_LITERAL
-    : 'year'
-    | 'month'
-    | 'day'
-    | 'second'
-    | 'minute'
-    | 'hour'
-    | 'week'
-    | 'millisecond'
-    | 'years'
-    | 'months'
-    | 'days'
-    | 'seconds'
-    | 'minutes'
-    | 'hours'
-    | 'weeks'
-    | 'milliseconds'
-    ;
+BY : B Y;
 
-AND : 'and';
+AND : A N D;
+ASC : A S C;
 ASSIGN : '=';
 COMMA : ',';
+DESC : D E S C;
 DOT : '.';
+FALSE : F A L S E;
+FIRST : F I R S T;
+LAST : L A S T;
 LP : '(';
-OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION);
-CLOSING_BRACKET : ']' -> popMode, popMode;
-NOT : N O T;
-LIKE: L I K E;
-RLIKE: R L I K E;
 IN: I N;
 IS: I S;
-AS: A S;
+LIKE: L I K E;
+NOT : N O T;
 NULL : N U L L;
-OR : 'or';
+NULLS : N U L L S;
+OR : O R;
+PARAM: '?';
+RLIKE: R L I K E;
 RP : ')';
+TRUE : T R U E;
+INFO : I N F O;
+FUNCTIONS : F U N C T I O N S;
 UNDERSCORE: '_';
-INFO : 'info';
-FUNCTIONS : 'functions';
-
-BOOLEAN_VALUE
-   : 'true'
-   | 'false'
-   ;
-
-COMPARISON_OPERATOR
-    : '=='
-    |'!='
-    | '<'
-    | '<='
-    | '>'
-    | '>='
-    ;
+
+EQ  : '==';
+NEQ : '!=';
+LT  : '<';
+LTE : '<=';
+GT  : '>';
+GTE : '>=';
 
 PLUS : '+';
 MINUS : '-';
 ASTERISK : '*';
 SLASH : '/';
 PERCENT : '%';
-TEN: '10';
-
-ORDERING
-    : 'asc'
-    | 'desc'
-    ;
-
-NULLS_ORDERING: 'nulls';
-NULLS_ORDERING_DIRECTION
-    : 'first'
-    | 'last'
-    ;
 
-MATH_FUNCTION
-    : R O U N D
-    | A B S
-    | P O W
-    | L O G TEN
-    | P I
-    | T A U
-    | E
-    | S U B S T R I N G
-    | T R I M
-    | C O N C A T
-    | C O A L E S C E
-    | G R E A T E S T
-    | L E F T
-    | N O W
-    | R I G H T
-    | S T A R T S UNDERSCORE W I T H
-    | D A T E UNDERSCORE F O R M A T
-    | D A T E UNDERSCORE T R U N C
-    | D A T E UNDERSCORE P A R S E
-    | A U T O UNDERSCORE B U C K E T
-    | D A T E UNDERSCORE E X T R A C T
-    | I S UNDERSCORE F I N I T E
-    | I S UNDERSCORE I N F I N I T E
-    | C A S E
-    | L E N G T H
-    | M V UNDERSCORE M A X
-    | M V UNDERSCORE M I N
-    | M V UNDERSCORE A V G
-    | M V UNDERSCORE S U M
-    | M V UNDERSCORE C O U N T
-    | M V UNDERSCORE C O N C A T
-    | M V UNDERSCORE J O I N
-    | M V UNDERSCORE M E D I A N
-    | M V UNDERSCORE D E D U P E
-    | M E T A D A T A
-    | S P L I T
-    | T O UNDERSCORE S T R I N G
-    | T O UNDERSCORE S T R
-    | T O UNDERSCORE B O O L
-    | T O UNDERSCORE B O O L E A N
-    | T O UNDERSCORE D A T E T I M E
-    | T O UNDERSCORE D T
-    | T O UNDERSCORE D B L
-    | T O UNDERSCORE D O U B L E
-    | T O UNDERSCORE D E G R E E S
-    | T O UNDERSCORE I N T
-    | T O UNDERSCORE I N T E G E R
-    | T O UNDERSCORE I P
-    | T O UNDERSCORE L O N G
-    | T O UNDERSCORE R A D I A N S
-    | T O UNDERSCORE V E R S I O N
-    | T O UNDERSCORE U N S I G N E D UNDERSCORE L O N G
-    ;
-
-UNARY_FUNCTION
-    : A V G
-    | M I N
-    | M A X
-    | S U M
-    | C O U N T
-    | C O U N T UNDERSCORE D I S T I N C T
-    | P E R C E N T I L E
-    | M E D I A N
-    | M E D I A N UNDERSCORE A B S O L U T E UNDERSCORE D E V I A T I O N
-    | A C O S
-    | A S I N
-    | A T A N
-    | A T A N '2'
-    | C E I L
-    | C O S
-    | C O S H
-    | F L O O R
-    | L T R I M
-    | S I N
-    | S I N H
-    | S Q R T
-    | T A N
-    | T A N H
-    ;
+// Brackets are funny. We can happen upon a CLOSING_BRACKET in two ways - one
+// way is to start in an explain command which then shifts us to expression
+// mode. Thus, the two popModes on CLOSING_BRACKET. The other way could as
+// the start of a multivalued field constant. To line up with the double pop
+// the explain mode needs, we double push when we see that.
+OPENING_BRACKET : '[' -> pushMode(EXPRESSION), pushMode(EXPRESSION);
+CLOSING_BRACKET : ']' -> popMode, popMode;
 
-WHERE_FUNCTIONS
-    : C I D R UNDERSCORE M A T C H
-    ;
 
 UNQUOTED_IDENTIFIER
-    : LETTER (LETTER | DIGIT | '_' | ASTERISK)*
+    : LETTER (LETTER | DIGIT | '_')*
     // only allow @ at beginning of identifier to keep the option to allow @ as infix operator in the future
     // also, single `_` and `@` characters are not valid identifiers
-    | ('_' | '@') (LETTER | DIGIT | '_' | ASTERISK)+
+    | ('_' | '@') (LETTER | DIGIT | '_')+
     ;
 
 QUOTED_IDENTIFIER
@@ -264,14 +151,18 @@ EXPR_WS
     ;
 
 
+
 mode SOURCE_IDENTIFIERS;
 
 SRC_PIPE : '|' -> type(PIPE), popMode;
 SRC_OPENING_BRACKET : '[' -> type(OPENING_BRACKET), pushMode(SOURCE_IDENTIFIERS), pushMode(SOURCE_IDENTIFIERS);
 SRC_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);
 SRC_COMMA : ',' -> type(COMMA);
 SRC_ASSIGN : '=' -> type(ASSIGN);
+AS : A S;
 METADATA: M E T A D A T A;
+ON : O N;
+WITH : W I T H;
 
 SRC_UNQUOTED_IDENTIFIER
     : SRC_UNQUOTED_IDENTIFIER_PART+
@@ -298,41 +189,6 @@ SRC_WS
     : WS -> channel(HIDDEN)
     ;
 
-mode ENRICH_IDENTIFIERS;
-
-ON : O N;
-WITH : W I T H;
-
-ENR_PIPE : '|' -> type(PIPE), popMode;
-ENR_CLOSING_BRACKET : ']' -> popMode, popMode, type(CLOSING_BRACKET);
-ENR_COMMA : ',' -> type(COMMA);
-ENR_ASSIGN : '=' -> type(ASSIGN);
-
-ENR_UNQUOTED_IDENTIFIER
-    : ENR_UNQUOTED_IDENTIFIER_PART+
-    ;
-
-fragment ENR_UNQUOTED_IDENTIFIER_PART
-    : ~[=`|,[\]/ \t\r\n]+
-    | '/' ~[*/] // allow single / but not followed by another / or * which would start a comment
-    ;
-
-ENR_QUOTED_IDENTIFIER
-    : QUOTED_IDENTIFIER
-    ;
-
-ENR_LINE_COMMENT
-    : LINE_COMMENT -> channel(HIDDEN)
-    ;
-
-ENR_MULTILINE_COMMENT
-    : MULTILINE_COMMENT -> channel(HIDDEN)
-    ;
-
-ENR_WS
-    : WS -> channel(HIDDEN)
-    ;
-
 fragment A : [aA]; // match either an 'a' or 'A'
 fragment B : [bB];
 fragment C : [cC];