diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 13c4ef0f60..38fca306bf 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,7 +27,11 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v3 - + - name: Set up JDK 21 + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 21 - name: Initialize CodeQL uses: github/codeql-action/init@v2 with: diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index 8ca5a5c1d5..77985193cb 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -20,7 +20,7 @@ jobs: strategy: fail-fast: false matrix: - java: [ 11, 17, 21 ] + java: [21] runs-on: ubuntu-latest container: @@ -60,7 +60,7 @@ jobs: fail-fast: false matrix: os: [ windows-latest ] - java: [ 11, 17, 21 ] + java: [21] runs-on: ${{ matrix.os }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index 8adf7ae52c..0dd07404bb 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/setup-java@v3 with: distribution: temurin # Temurin is a distribution of adoptium - java-version: 11 + java-version: 21 - uses: actions/checkout@v3 - uses: aws-actions/configure-aws-credentials@v1.7.0 with: diff --git a/.github/workflows/sql-pitest.yml b/.github/workflows/sql-pitest.yml index bc751daefa..a1281a67f3 100644 --- a/.github/workflows/sql-pitest.yml +++ b/.github/workflows/sql-pitest.yml @@ -21,8 +21,7 @@ jobs: strategy: matrix: java: - - 11 - - 17 + - 21 runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index ded974bfcb..042244a4c0 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -28,10 +28,7 @@ jobs: # Run all jobs fail-fast: false matrix: - java: - - 11 - - 17 - - 21 + java: [21] runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution @@ -105,8 +102,6 @@ jobs: fail-fast: false matrix: entry: - - { os: windows-latest, java: 11, os_build_args: -x doctest -PbuildPlatform=windows } - - { os: windows-latest, java: 17, os_build_args: -x doctest -PbuildPlatform=windows } - { os: windows-latest, java: 21, os_build_args: -x doctest -PbuildPlatform=windows } runs-on: ${{ matrix.entry.os }} diff --git a/.github/workflows/sql-test-workflow.yml b/.github/workflows/sql-test-workflow.yml index 46c1930cc8..c447d968bd 100644 --- a/.github/workflows/sql-test-workflow.yml +++ b/.github/workflows/sql-test-workflow.yml @@ -21,8 +21,7 @@ jobs: strategy: matrix: java: - - 11 - - 17 + - 21 runs-on: ubuntu-latest container: # using the same image which is used by opensearch-build team to build the OpenSearch Distribution diff --git a/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 b/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 index dc097d596d..2e8d634dad 100644 --- a/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 +++ b/async-query-core/src/main/antlr/FlintSparkSqlExtensions.g4 @@ -188,7 +188,7 @@ indexColTypeList ; indexColType - : identifier skipType=(PARTITION | VALUE_SET | MIN_MAX | BLOOM_FILTER) + : multipartIdentifier skipType=(PARTITION | VALUE_SET | MIN_MAX | BLOOM_FILTER) (LEFT_PAREN skipParams RIGHT_PAREN)? ; diff --git a/async-query-core/src/main/antlr/SqlBaseLexer.g4 b/async-query-core/src/main/antlr/SqlBaseLexer.g4 index a9705c1733..85a4633e80 100644 --- a/async-query-core/src/main/antlr/SqlBaseLexer.g4 +++ b/async-query-core/src/main/antlr/SqlBaseLexer.g4 @@ -134,6 +134,7 @@ AS: 'AS'; ASC: 'ASC'; AT: 'AT'; AUTHORIZATION: 'AUTHORIZATION'; +BEGIN: 'BEGIN'; BETWEEN: 'BETWEEN'; BIGINT: 'BIGINT'; BINARY: 'BINARY'; @@ -145,6 +146,7 @@ BUCKETS: 'BUCKETS'; BY: 'BY'; BYTE: 'BYTE'; CACHE: 'CACHE'; +CALLED: 'CALLED'; CASCADE: 'CASCADE'; CASE: 'CASE'; CAST: 'CAST'; @@ -171,6 +173,7 @@ COMPENSATION: 'COMPENSATION'; COMPUTE: 'COMPUTE'; CONCATENATE: 'CONCATENATE'; CONSTRAINT: 'CONSTRAINT'; +CONTAINS: 'CONTAINS'; COST: 'COST'; CREATE: 'CREATE'; CROSS: 'CROSS'; @@ -197,10 +200,12 @@ DECIMAL: 'DECIMAL'; DECLARE: 'DECLARE'; DEFAULT: 'DEFAULT'; DEFINED: 'DEFINED'; +DEFINER: 'DEFINER'; DELETE: 'DELETE'; DELIMITED: 'DELIMITED'; DESC: 'DESC'; DESCRIBE: 'DESCRIBE'; +DETERMINISTIC: 'DETERMINISTIC'; DFS: 'DFS'; DIRECTORIES: 'DIRECTORIES'; DIRECTORY: 'DIRECTORY'; @@ -259,6 +264,7 @@ INDEX: 'INDEX'; INDEXES: 'INDEXES'; INNER: 'INNER'; INPATH: 'INPATH'; +INPUT: 'INPUT'; INPUTFORMAT: 'INPUTFORMAT'; INSERT: 'INSERT'; INTERSECT: 'INTERSECT'; @@ -266,10 +272,12 @@ INTERVAL: 'INTERVAL'; INT: 'INT'; INTEGER: 'INTEGER'; INTO: 'INTO'; +INVOKER: 'INVOKER'; IS: 'IS'; ITEMS: 'ITEMS'; JOIN: 'JOIN'; KEYS: 'KEYS'; +LANGUAGE: 'LANGUAGE'; LAST: 'LAST'; LATERAL: 'LATERAL'; LAZY: 'LAZY'; @@ -297,6 +305,7 @@ MILLISECOND: 'MILLISECOND'; MILLISECONDS: 'MILLISECONDS'; MINUTE: 'MINUTE'; MINUTES: 'MINUTES'; +MODIFIES: 'MODIFIES'; MONTH: 'MONTH'; MONTHS: 'MONTHS'; MSCK: 'MSCK'; @@ -341,6 +350,7 @@ PURGE: 'PURGE'; QUARTER: 'QUARTER'; QUERY: 'QUERY'; RANGE: 'RANGE'; +READS: 'READS'; REAL: 'REAL'; RECORDREADER: 'RECORDREADER'; RECORDWRITER: 'RECORDWRITER'; @@ -355,6 +365,8 @@ REPLACE: 'REPLACE'; RESET: 'RESET'; RESPECT: 'RESPECT'; RESTRICT: 'RESTRICT'; +RETURN: 'RETURN'; +RETURNS: 'RETURNS'; REVOKE: 'REVOKE'; RIGHT: 'RIGHT'; RLIKE: 'RLIKE' | 'REGEXP'; @@ -368,6 +380,7 @@ SECOND: 'SECOND'; SECONDS: 'SECONDS'; SCHEMA: 'SCHEMA'; SCHEMAS: 'SCHEMAS'; +SECURITY: 'SECURITY'; SELECT: 'SELECT'; SEMI: 'SEMI'; SEPARATED: 'SEPARATED'; @@ -386,6 +399,8 @@ SOME: 'SOME'; SORT: 'SORT'; SORTED: 'SORTED'; SOURCE: 'SOURCE'; +SPECIFIC: 'SPECIFIC'; +SQL: 'SQL'; START: 'START'; STATISTICS: 'STATISTICS'; STORED: 'STORED'; diff --git a/async-query-core/src/main/antlr/SqlBaseParser.g4 b/async-query-core/src/main/antlr/SqlBaseParser.g4 index 4552c17e0c..54eff14b6d 100644 --- a/async-query-core/src/main/antlr/SqlBaseParser.g4 +++ b/async-query-core/src/main/antlr/SqlBaseParser.g4 @@ -42,6 +42,28 @@ options { tokenVocab = SqlBaseLexer; } public boolean double_quoted_identifiers = false; } +compoundOrSingleStatement + : singleStatement + | singleCompoundStatement + ; + +singleCompoundStatement + : beginEndCompoundBlock SEMICOLON? EOF + ; + +beginEndCompoundBlock + : BEGIN compoundBody END + ; + +compoundBody + : (compoundStatements+=compoundStatement SEMICOLON)* + ; + +compoundStatement + : statement + | beginEndCompoundBlock + ; + singleStatement : statement SEMICOLON* EOF ; @@ -83,13 +105,15 @@ statement (WITH (DBPROPERTIES | PROPERTIES) propertyList))* #createNamespace | ALTER namespace identifierReference SET (DBPROPERTIES | PROPERTIES) propertyList #setNamespaceProperties + | ALTER namespace identifierReference + UNSET (DBPROPERTIES | PROPERTIES) propertyList #unsetNamespaceProperties | ALTER namespace identifierReference SET locationSpec #setNamespaceLocation | DROP namespace (IF EXISTS)? identifierReference (RESTRICT | CASCADE)? #dropNamespace | SHOW namespaces ((FROM | IN) multipartIdentifier)? (LIKE? pattern=stringLit)? #showNamespaces - | createTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? + | createTableHeader (LEFT_PAREN colDefinitionList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #createTable | CREATE TABLE (IF errorCapturingNot EXISTS)? target=tableIdentifier @@ -99,7 +123,7 @@ statement createFileFormat | locationSpec | (TBLPROPERTIES tableProps=propertyList))* #createTableLike - | replaceTableHeader (LEFT_PAREN createOrReplaceTableColTypeList RIGHT_PAREN)? tableProvider? + | replaceTableHeader (LEFT_PAREN colDefinitionList RIGHT_PAREN)? tableProvider? createTableClauses (AS? query)? #replaceTable | ANALYZE TABLE identifierReference partitionSpec? COMPUTE STATISTICS @@ -168,6 +192,11 @@ statement | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF errorCapturingNot EXISTS)? identifierReference AS className=stringLit (USING resource (COMMA resource)*)? #createFunction + | CREATE (OR REPLACE)? TEMPORARY? FUNCTION (IF errorCapturingNot EXISTS)? + identifierReference LEFT_PAREN parameters=colDefinitionList? RIGHT_PAREN + (RETURNS (dataType | TABLE LEFT_PAREN returnParams=colTypeList RIGHT_PAREN))? + routineCharacteristics + RETURN (query | expression) #createUserDefinedFunction | DROP TEMPORARY? FUNCTION (IF EXISTS)? identifierReference #dropFunction | DECLARE (OR REPLACE)? VARIABLE? identifierReference dataType? variableDefaultExpression? #createVariable @@ -1186,11 +1215,11 @@ colType : colName=errorCapturingIdentifier dataType (errorCapturingNot NULL)? commentSpec? ; -createOrReplaceTableColTypeList - : createOrReplaceTableColType (COMMA createOrReplaceTableColType)* +colDefinitionList + : colDefinition (COMMA colDefinition)* ; -createOrReplaceTableColType +colDefinition : colName=errorCapturingIdentifier dataType colDefinitionOption* ; @@ -1213,6 +1242,46 @@ complexColType : errorCapturingIdentifier COLON? dataType (errorCapturingNot NULL)? commentSpec? ; +routineCharacteristics + : (routineLanguage + | specificName + | deterministic + | sqlDataAccess + | nullCall + | commentSpec + | rightsClause)* + ; + +routineLanguage + : LANGUAGE (SQL | IDENTIFIER) + ; + +specificName + : SPECIFIC specific=errorCapturingIdentifier + ; + +deterministic + : DETERMINISTIC + | errorCapturingNot DETERMINISTIC + ; + +sqlDataAccess + : access=NO SQL + | access=CONTAINS SQL + | access=READS SQL DATA + | access=MODIFIES SQL DATA + ; + +nullCall + : RETURNS NULL ON NULL INPUT + | CALLED ON NULL INPUT + ; + +rightsClause + : SQL SECURITY INVOKER + | SQL SECURITY DEFINER + ; + whenClause : WHEN condition=expression THEN result=expression ; @@ -1360,6 +1429,7 @@ ansiNonReserved | ARRAY | ASC | AT + | BEGIN | BETWEEN | BIGINT | BINARY @@ -1371,6 +1441,7 @@ ansiNonReserved | BY | BYTE | CACHE + | CALLED | CASCADE | CATALOG | CATALOGS @@ -1390,6 +1461,7 @@ ansiNonReserved | COMPENSATION | COMPUTE | CONCATENATE + | CONTAINS | COST | CUBE | CURRENT @@ -1410,10 +1482,12 @@ ansiNonReserved | DECLARE | DEFAULT | DEFINED + | DEFINER | DELETE | DELIMITED | DESC | DESCRIBE + | DETERMINISTIC | DFS | DIRECTORIES | DIRECTORY @@ -1454,13 +1528,16 @@ ansiNonReserved | INDEX | INDEXES | INPATH + | INPUT | INPUTFORMAT | INSERT | INT | INTEGER | INTERVAL + | INVOKER | ITEMS | KEYS + | LANGUAGE | LAST | LAZY | LIKE @@ -1485,6 +1562,7 @@ ansiNonReserved | MILLISECONDS | MINUTE | MINUTES + | MODIFIES | MONTH | MONTHS | MSCK @@ -1518,6 +1596,7 @@ ansiNonReserved | QUARTER | QUERY | RANGE + | READS | REAL | RECORDREADER | RECORDWRITER @@ -1531,6 +1610,8 @@ ansiNonReserved | RESET | RESPECT | RESTRICT + | RETURN + | RETURNS | REVOKE | RLIKE | ROLE @@ -1543,6 +1624,7 @@ ansiNonReserved | SCHEMAS | SECOND | SECONDS + | SECURITY | SEMI | SEPARATED | SERDE @@ -1558,6 +1640,7 @@ ansiNonReserved | SORT | SORTED | SOURCE + | SPECIFIC | START | STATISTICS | STORED @@ -1662,6 +1745,7 @@ nonReserved | ASC | AT | AUTHORIZATION + | BEGIN | BETWEEN | BIGINT | BINARY @@ -1674,6 +1758,7 @@ nonReserved | BY | BYTE | CACHE + | CALLED | CASCADE | CASE | CAST @@ -1700,6 +1785,7 @@ nonReserved | COMPUTE | CONCATENATE | CONSTRAINT + | CONTAINS | COST | CREATE | CUBE @@ -1725,10 +1811,12 @@ nonReserved | DECLARE | DEFAULT | DEFINED + | DEFINER | DELETE | DELIMITED | DESC | DESCRIBE + | DETERMINISTIC | DFS | DIRECTORIES | DIRECTORY @@ -1784,15 +1872,18 @@ nonReserved | INDEX | INDEXES | INPATH + | INPUT | INPUTFORMAT | INSERT | INT | INTEGER | INTERVAL | INTO + | INVOKER | IS | ITEMS | KEYS + | LANGUAGE | LAST | LAZY | LEADING @@ -1819,6 +1910,7 @@ nonReserved | MILLISECONDS | MINUTE | MINUTES + | MODIFIES | MONTH | MONTHS | MSCK @@ -1861,6 +1953,7 @@ nonReserved | QUARTER | QUERY | RANGE + | READS | REAL | RECORDREADER | RECORDWRITER @@ -1875,6 +1968,8 @@ nonReserved | RESET | RESPECT | RESTRICT + | RETURN + | RETURNS | REVOKE | RLIKE | ROLE @@ -1887,6 +1982,7 @@ nonReserved | SCHEMAS | SECOND | SECONDS + | SECURITY | SELECT | SEPARATED | SERDE @@ -1903,6 +1999,8 @@ nonReserved | SORT | SORTED | SOURCE + | SPECIFIC + | SQL | START | STATISTICS | STORED diff --git a/build.gradle b/build.gradle index 3e180d4042..b3e09d7b50 100644 --- a/build.gradle +++ b/build.gradle @@ -112,7 +112,7 @@ allprojects { } plugins.withId('java') { - sourceCompatibility = targetCompatibility = "11" + sourceCompatibility = targetCompatibility = JavaVersion.VERSION_21 } configurations.all { resolutionStrategy.force "org.jetbrains.kotlin:kotlin-stdlib:1.9.10" diff --git a/integ-test/build.gradle b/integ-test/build.gradle index a0f82b6f1b..93153cf737 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -44,7 +44,7 @@ apply plugin: 'java' apply plugin: 'io.freefair.lombok' apply plugin: 'com.wiredforcode.spawn' -String baseVersion = "2.15.0" +String baseVersion = "2.16.0" String bwcVersion = baseVersion + ".0"; String baseName = "sqlBwcCluster" String bwcFilePath = "src/test/resources/bwc/" @@ -194,6 +194,11 @@ dependencies { zipArchive group: 'org.opensearch.plugin', name:'opensearch-sql-plugin', version: "${bwcVersion}-SNAPSHOT" } +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + dependencyLicenses.enabled = false testingConventions.enabled = false forbiddenApisTest.enabled = false diff --git a/plugin/build.gradle b/plugin/build.gradle index ed6fe53247..710d81ed0a 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -172,6 +172,11 @@ ext { noticeFile = rootProject.file('NOTICE') } +java { + sourceCompatibility = JavaVersion.VERSION_21 + targetCompatibility = JavaVersion.VERSION_21 +} + // ANTLR generated parser file is too large to be checked which caused licenseHeaders stuck. licenseHeaders { enabled = true @@ -282,3 +287,4 @@ testClusters.integTest { run { useCluster testClusters.integTest } +