From cd67383f41bfb7d69bcf3fc088bcbc6292465105 Mon Sep 17 00:00:00 2001 From: Johannes Feichtner Date: Sun, 27 Nov 2022 20:38:55 +0100 Subject: [PATCH 1/5] feat(manager/gradle): add support for Groovy and Kotlin maps --- lib/modules/manager/gradle/parser.spec.ts | 181 ++++++++++++++++++ lib/modules/manager/gradle/parser.ts | 128 +++++++++++++ lib/modules/manager/gradle/parser/handlers.ts | 38 ++-- 3 files changed, 332 insertions(+), 15 deletions(-) diff --git a/lib/modules/manager/gradle/parser.spec.ts b/lib/modules/manager/gradle/parser.spec.ts index c524baf5913cf9..d562a9767d7c90 100644 --- a/lib/modules/manager/gradle/parser.spec.ts +++ b/lib/modules/manager/gradle/parser.spec.ts @@ -59,6 +59,105 @@ describe('modules/manager/gradle/parser', () => { }); }); + describe('Groovy: multi var assignments', () => { + it('simple map', () => { + const input = ` + ext { + versions = [ + spotbugs_annotations : '4.5.3', + core : '1.7.0', + ] + + ignored = [ 'asdf' ] + + libraries = [:] + libraries += [ + guava: "com.google.guava:guava:31.1-jre", + detekt: '1.18.1', + ] + } + `; + + const output = { + 'versions.spotbugs_annotations': '4.5.3', + 'versions.core': '1.7.0', + 'libraries.guava': 'com.google.guava:guava:31.1-jre', + 'libraries.detekt': '1.18.1', + }; + + const { vars } = parseGradle(input); + for (const [key, value] of Object.entries(output)) { + expect(vars).toContainKey(key); + expect(vars[key]).toMatchObject({ key, value }); + } + }); + + it('nested map', () => { + const input = ` + project.ext.versions = [ + some: invalidsymbol, + android: [ + buildTools: '30.0.3' + ], + kotlin: '1.4.30', + androidx: [ + paging: '2.1.2', + kotlin: [ + stdlib: '1.4.20', + coroutines: '1.3.7', + ], + ], + espresso: '3.2.0' + ] + `; + + const output = { + 'versions.android.buildTools': '30.0.3', + 'versions.kotlin': '1.4.30', + 'versions.androidx.paging': '2.1.2', + 'versions.androidx.kotlin.stdlib': '1.4.20', + 'versions.androidx.kotlin.coroutines': '1.3.7', + 'versions.espresso': '3.2.0', + }; + + const { vars } = parseGradle(input); + for (const [key, value] of Object.entries(output)) { + expect(vars).toContainKey(key); + expect(vars[key]).toMatchObject({ key, value }); + } + }); + + it('map with interpolated dependency strings', () => { + const input = ` + def slfj4Version = "2.0.0" + libraries = [ + jcl: "org.slf4j:jcl-over-slf4j:\${slfj4Version}", + releaseCoroutines: "org.jetbrains.kotlinx:kotlinx-coroutines-core:0.26.1-eap13" + api: "org.slf4j:slf4j-api:$slfj4Version", + ] + `; + + const { deps } = parseGradle(input); + expect(deps).toMatchObject([ + { + depName: 'org.slf4j:jcl-over-slf4j', + groupName: 'slfj4Version', + currentValue: '2.0.0', + }, + { + depName: 'org.jetbrains.kotlinx:kotlinx-coroutines-core', + groupName: 'libraries.releaseCoroutines', + currentValue: '0.26.1-eap13', + }, + { + depName: 'org.slf4j:slf4j-api', + groupName: 'slfj4Version', + currentValue: '2.0.0', + }, + ]); + }); + }); + describe('Kotlin: single var assignments', () => { test.each` input | name | value @@ -81,6 +180,88 @@ describe('modules/manager/gradle/parser', () => { expect(vars).toBeEmpty(); }); }); + + describe('Kotlin: multi var assignments', () => { + it('simple map', () => { + const input = + 'val versions = mapOf("foo1" to "bar1", "foo2" to "bar2", "foo3" to "bar3")'; + const output = { + 'versions.foo1': 'bar1', + 'versions.foo2': 'bar2', + 'versions.foo3': 'bar3', + }; + + const { vars } = parseGradle(input); + for (const [key, value] of Object.entries(output)) { + expect(vars).toContainKey(key); + expect(vars[key]).toMatchObject({ key, value }); + } + }); + + it('nested map', () => { + const input = ` + ext["deps"] = mapOf( + "support" to mapOf( + "appCompat" to "com.android.support:appcompat-v7:26.0.2", + "invalid" to whatever, + "junit" to mapOf( + "jupiter" to "5.0.1", + "platform" to "1.0.1", + ) + "design" to "com.android.support:design:26.0.2" + ), + "support2" to mapOfInvalid( + "design2" to "com.android.support:design:26.0.2" + ), + "picasso" to "com.squareup.picasso:picasso:2.5.2" + ) + `; + + const output = { + 'deps.support.appCompat': 'com.android.support:appcompat-v7:26.0.2', + 'deps.support.design': 'com.android.support:design:26.0.2', + 'deps.support.junit.jupiter': '5.0.1', + 'deps.support.junit.platform': '1.0.1', + 'deps.picasso': 'com.squareup.picasso:picasso:2.5.2', + }; + + const { vars } = parseGradle(input); + for (const [key, value] of Object.entries(output)) { + expect(vars).toContainKey(key); + expect(vars[key]).toMatchObject({ key, value }); + } + }); + + it('map with interpolated dependency strings', () => { + const input = ` + val slfj4Version = "2.0.0" + libraries = mapOf( + "jcl" to "org.slf4j:jcl-over-slf4j:\${slfj4Version}", + "releaseCoroutines" to "org.jetbrains.kotlinx:kotlinx-coroutines-core:0.26.1-eap13" + "api" to "org.slf4j:slf4j-api:$slfj4Version", + ) + `; + + const { deps } = parseGradle(input); + expect(deps).toMatchObject([ + { + depName: 'org.slf4j:jcl-over-slf4j', + groupName: 'slfj4Version', + currentValue: '2.0.0', + }, + { + depName: 'org.jetbrains.kotlinx:kotlinx-coroutines-core', + groupName: 'libraries.releaseCoroutines', + currentValue: '0.26.1-eap13', + }, + { + depName: 'org.slf4j:slf4j-api', + groupName: 'slfj4Version', + currentValue: '2.0.0', + }, + ]); + }); + }); }); describe('dependencies', () => { diff --git a/lib/modules/manager/gradle/parser.ts b/lib/modules/manager/gradle/parser.ts index 9c25dd3f81f541..1c146e78805308 100644 --- a/lib/modules/manager/gradle/parser.ts +++ b/lib/modules/manager/gradle/parser.ts @@ -147,6 +147,132 @@ const qKotlinSingleVarAssignment = q }) .handler(cleanupTempVars); +// foo: "1.2.3" +const qGroovySingleMapOfVarAssignment = q + .sym(storeVarToken) + .handler((ctx) => { + ctx.tmpTokenStore.backupVarTokens = ctx.varTokens; + return ctx; + }) + .handler(coalesceVariable) + .handler((ctx) => storeInTokenMap(ctx, 'keyToken')) + .op(':') + .join(qTemplateString) + .handler((ctx) => storeInTokenMap(ctx, 'valToken')) + .handler(handleAssignment) + .handler((ctx) => { + ctx.varTokens = ctx.tmpTokenStore.backupVarTokens!; + ctx.varTokens.pop(); + return ctx; + }); + +// versions = [ android: [ buildTools: '30.0.3' ], kotlin: '1.4.30' ] +const qGroovyMultiVarAssignment = qVariableAssignmentIdentifier + .alt(q.op('='), q.op('+=')) + .tree({ + type: 'wrapped-tree', + maxDepth: 1, + startsWith: '[', + endsWith: ']', + search: q.alt( + q + .sym(storeVarToken) + .op(':') + .tree({ + type: 'wrapped-tree', + maxDepth: 1, + startsWith: '[', + endsWith: ']', + search: q.alt( + q + .sym(storeVarToken) + .op(':') + .tree({ + type: 'wrapped-tree', + maxDepth: 1, + startsWith: '[', + endsWith: ']', + search: qGroovySingleMapOfVarAssignment, + postHandler: (ctx) => { + ctx.varTokens.pop(); + return ctx; + }, + }), + qGroovySingleMapOfVarAssignment + ), + postHandler: (ctx) => { + ctx.varTokens.pop(); + return ctx; + }, + }), + qGroovySingleMapOfVarAssignment + ), + }) + .handler(cleanupTempVars); + +// "foo1" to "bar1" +const qKotlinSingleMapOfVarAssignment = qStringValue + .sym('to') + .handler((ctx) => { + ctx.tmpTokenStore.backupVarTokens = ctx.varTokens; + return ctx; + }) + .handler(coalesceVariable) + .handler((ctx) => storeInTokenMap(ctx, 'keyToken')) + .join(qTemplateString) + .handler((ctx) => storeInTokenMap(ctx, 'valToken')) + .handler(handleAssignment) + .handler((ctx) => { + ctx.varTokens = ctx.tmpTokenStore.backupVarTokens!; + ctx.varTokens.pop(); + return ctx; + }); + +// val versions = mapOf("foo1" to "bar1", "foo2" to "bar2", "foo3" to "bar3") +const qKotlinMultiMapOfVarAssignment = qVariableAssignmentIdentifier + .op('=') + .sym('mapOf') + .tree({ + type: 'wrapped-tree', + maxDepth: 1, + startsWith: '(', + endsWith: ')', + search: q.alt( + qStringValue + .sym('to') + .sym('mapOf') + .tree({ + type: 'wrapped-tree', + maxDepth: 1, + startsWith: '(', + endsWith: ')', + search: q.alt( + qStringValue + .sym('to') + .sym('mapOf') + .tree({ + type: 'wrapped-tree', + maxDepth: 1, + startsWith: '(', + endsWith: ')', + search: qKotlinSingleMapOfVarAssignment, + postHandler: (ctx) => { + ctx.varTokens.pop(); + return ctx; + }, + }), + qKotlinSingleMapOfVarAssignment + ), + postHandler: (ctx) => { + ctx.varTokens.pop(); + return ctx; + }, + }), + qKotlinSingleMapOfVarAssignment + ), + }) + .handler(cleanupTempVars); + // "foo:bar:1.2.3" const qDependenciesSimpleString = qStringValue .handler((ctx) => storeInTokenMap(ctx, 'stringToken')) @@ -493,7 +619,9 @@ export function parseGradle( maxDepth: 32, search: q.alt( qGroovySingleVarAssignment, + qGroovyMultiVarAssignment, qKotlinSingleVarAssignment, + qKotlinMultiMapOfVarAssignment, qDependenciesSimpleString, qDependenciesInterpolation, qGroovyMapNotationDependencies, diff --git a/lib/modules/manager/gradle/parser/handlers.ts b/lib/modules/manager/gradle/parser/handlers.ts index 94790307718351..58272e3e8a8993 100644 --- a/lib/modules/manager/gradle/parser/handlers.ts +++ b/lib/modules/manager/gradle/parser/handlers.ts @@ -16,26 +16,34 @@ import { export function handleAssignment(ctx: Ctx): Ctx { const key = loadFromTokenMap(ctx, 'keyToken')[0].value; - const valToken = loadFromTokenMap(ctx, 'valToken')[0]; + const valTokens = loadFromTokenMap(ctx, 'valToken'); - const dep = parseDependencyString(valToken.value); - if (dep) { - dep.groupName = key; - dep.managerData = { - fileReplacePosition: valToken.offset + dep.depName!.length + 1, + if (valTokens.length > 1) { + // = template string with multiple variables + ctx.tokenMap.templateStringTokens = valTokens; + handleDepInterpolation(ctx); + delete ctx.tokenMap.templateStringTokens; + } else { + // = string value + const dep = parseDependencyString(valTokens[0].value); + if (dep) { + dep.groupName = key; + dep.managerData = { + fileReplacePosition: valTokens[0].offset + dep.depName!.length + 1, + packageFile: ctx.packageFile, + }; + ctx.deps.push(dep); + } + + const varData: VariableData = { + key, + value: valTokens[0].value, + fileReplacePosition: valTokens[0].offset, packageFile: ctx.packageFile, }; - ctx.deps.push(dep); + ctx.globalVars = { ...ctx.globalVars, [key]: varData }; } - const varData: VariableData = { - key, - value: valToken.value, - fileReplacePosition: valToken.offset, - packageFile: ctx.packageFile, - }; - ctx.globalVars = { ...ctx.globalVars, [key]: varData }; - return ctx; } From 670e46fa394dcb05952efad8bd7d1dd54d9a5d77 Mon Sep 17 00:00:00 2001 From: Johannes Feichtner Date: Sun, 27 Nov 2022 23:48:54 +0100 Subject: [PATCH 2/5] tests: replace for loops with single object match --- lib/modules/manager/gradle/parser.spec.ts | 68 +++++++++++++++-------- 1 file changed, 44 insertions(+), 24 deletions(-) diff --git a/lib/modules/manager/gradle/parser.spec.ts b/lib/modules/manager/gradle/parser.spec.ts index d562a9767d7c90..da49fc11b0dc4c 100644 --- a/lib/modules/manager/gradle/parser.spec.ts +++ b/lib/modules/manager/gradle/parser.spec.ts @@ -78,18 +78,25 @@ describe('modules/manager/gradle/parser', () => { } `; - const output = { - 'versions.spotbugs_annotations': '4.5.3', - 'versions.core': '1.7.0', - 'libraries.guava': 'com.google.guava:guava:31.1-jre', - 'libraries.detekt': '1.18.1', - }; - const { vars } = parseGradle(input); - for (const [key, value] of Object.entries(output)) { - expect(vars).toContainKey(key); - expect(vars[key]).toMatchObject({ key, value }); - } + expect(vars).toMatchObject({ + 'versions.spotbugs_annotations': { + key: 'versions.spotbugs_annotations', + value: '4.5.3', + }, + 'versions.core': { + key: 'versions.core', + value: '1.7.0', + }, + 'libraries.guava': { + key: 'libraries.guava', + value: 'com.google.guava:guava:31.1-jre', + }, + 'libraries.detekt': { + key: 'libraries.detekt', + value: '1.18.1', + }, + }); }); it('nested map', () => { @@ -111,20 +118,33 @@ describe('modules/manager/gradle/parser', () => { ] `; - const output = { - 'versions.android.buildTools': '30.0.3', - 'versions.kotlin': '1.4.30', - 'versions.androidx.paging': '2.1.2', - 'versions.androidx.kotlin.stdlib': '1.4.20', - 'versions.androidx.kotlin.coroutines': '1.3.7', - 'versions.espresso': '3.2.0', - }; - const { vars } = parseGradle(input); - for (const [key, value] of Object.entries(output)) { - expect(vars).toContainKey(key); - expect(vars[key]).toMatchObject({ key, value }); - } + expect(vars).toMatchObject({ + 'versions.android.buildTools': { + key: 'versions.android.buildTools', + value: '30.0.3', + }, + 'versions.kotlin': { + key: 'versions.kotlin', + value: '1.4.30', + }, + 'versions.androidx.paging': { + key: 'versions.androidx.paging', + value: '2.1.2', + }, + 'versions.androidx.kotlin.stdlib': { + key: 'versions.androidx.kotlin.stdlib', + value: '1.4.20', + }, + 'versions.androidx.kotlin.coroutines': { + key: 'versions.androidx.kotlin.coroutines', + value: '1.3.7', + }, + 'versions.espresso': { + key: 'versions.espresso', + value: '3.2.0', + }, + }); }); it('map with interpolated dependency strings', () => { From 48bd560d0d8208188c689c78a1e9da7f3c51a032 Mon Sep 17 00:00:00 2001 From: Michael Kriese Date: Mon, 28 Nov 2022 10:56:18 +0100 Subject: [PATCH 3/5] Update lib/modules/manager/gradle/parser/handlers.ts Co-authored-by: Sergei Zharinov --- lib/modules/manager/gradle/parser/handlers.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/manager/gradle/parser/handlers.ts b/lib/modules/manager/gradle/parser/handlers.ts index 58272e3e8a8993..4005b6a8f4a32f 100644 --- a/lib/modules/manager/gradle/parser/handlers.ts +++ b/lib/modules/manager/gradle/parser/handlers.ts @@ -41,7 +41,7 @@ export function handleAssignment(ctx: Ctx): Ctx { fileReplacePosition: valTokens[0].offset, packageFile: ctx.packageFile, }; - ctx.globalVars = { ...ctx.globalVars, [key]: varData }; + ctx.globalVars[key] = varData; } return ctx; From e9314efd298ef76122c146f7f4079e7e728eab2b Mon Sep 17 00:00:00 2001 From: Johannes Feichtner Date: Mon, 28 Nov 2022 19:37:25 +0100 Subject: [PATCH 4/5] codeBlock it all --- lib/modules/manager/gradle/parser.spec.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/modules/manager/gradle/parser.spec.ts b/lib/modules/manager/gradle/parser.spec.ts index da49fc11b0dc4c..d9af6d856405f1 100644 --- a/lib/modules/manager/gradle/parser.spec.ts +++ b/lib/modules/manager/gradle/parser.spec.ts @@ -1,3 +1,4 @@ +import { codeBlock } from 'common-tags'; import { Fixtures } from '../../../../test/fixtures'; import { fs, logger } from '../../../../test/util'; import { parseGradle, parseProps } from './parser'; @@ -61,7 +62,7 @@ describe('modules/manager/gradle/parser', () => { describe('Groovy: multi var assignments', () => { it('simple map', () => { - const input = ` + const input = codeBlock` ext { versions = [ spotbugs_annotations : '4.5.3', @@ -100,7 +101,7 @@ describe('modules/manager/gradle/parser', () => { }); it('nested map', () => { - const input = ` + const input = codeBlock` project.ext.versions = [ some: invalidsymbol, android: [ @@ -148,7 +149,7 @@ describe('modules/manager/gradle/parser', () => { }); it('map with interpolated dependency strings', () => { - const input = ` + const input = codeBlock` def slfj4Version = "2.0.0" libraries = [ jcl: "org.slf4j:jcl-over-slf4j:\${slfj4Version}", @@ -219,7 +220,7 @@ describe('modules/manager/gradle/parser', () => { }); it('nested map', () => { - const input = ` + const input = codeBlock` ext["deps"] = mapOf( "support" to mapOf( "appCompat" to "com.android.support:appcompat-v7:26.0.2", @@ -253,7 +254,7 @@ describe('modules/manager/gradle/parser', () => { }); it('map with interpolated dependency strings', () => { - const input = ` + const input = codeBlock` val slfj4Version = "2.0.0" libraries = mapOf( "jcl" to "org.slf4j:jcl-over-slf4j:\${slfj4Version}", From 2a2b90599c2ca07b06ad0876e806ec07b670d428 Mon Sep 17 00:00:00 2001 From: Johannes Feichtner Date: Mon, 28 Nov 2022 20:11:37 +0100 Subject: [PATCH 5/5] tests: replace for loops with single object match (act II) --- lib/modules/manager/gradle/parser.spec.ts | 57 ++++++++++++++--------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/lib/modules/manager/gradle/parser.spec.ts b/lib/modules/manager/gradle/parser.spec.ts index d9af6d856405f1..c7d33c59375ed5 100644 --- a/lib/modules/manager/gradle/parser.spec.ts +++ b/lib/modules/manager/gradle/parser.spec.ts @@ -206,17 +206,22 @@ describe('modules/manager/gradle/parser', () => { it('simple map', () => { const input = 'val versions = mapOf("foo1" to "bar1", "foo2" to "bar2", "foo3" to "bar3")'; - const output = { - 'versions.foo1': 'bar1', - 'versions.foo2': 'bar2', - 'versions.foo3': 'bar3', - }; const { vars } = parseGradle(input); - for (const [key, value] of Object.entries(output)) { - expect(vars).toContainKey(key); - expect(vars[key]).toMatchObject({ key, value }); - } + expect(vars).toMatchObject({ + 'versions.foo1': { + key: 'versions.foo1', + value: 'bar1', + }, + 'versions.foo2': { + key: 'versions.foo2', + value: 'bar2', + }, + 'versions.foo3': { + key: 'versions.foo3', + value: 'bar3', + }, + }); }); it('nested map', () => { @@ -238,19 +243,29 @@ describe('modules/manager/gradle/parser', () => { ) `; - const output = { - 'deps.support.appCompat': 'com.android.support:appcompat-v7:26.0.2', - 'deps.support.design': 'com.android.support:design:26.0.2', - 'deps.support.junit.jupiter': '5.0.1', - 'deps.support.junit.platform': '1.0.1', - 'deps.picasso': 'com.squareup.picasso:picasso:2.5.2', - }; - const { vars } = parseGradle(input); - for (const [key, value] of Object.entries(output)) { - expect(vars).toContainKey(key); - expect(vars[key]).toMatchObject({ key, value }); - } + expect(vars).toMatchObject({ + 'deps.support.appCompat': { + key: 'deps.support.appCompat', + value: 'com.android.support:appcompat-v7:26.0.2', + }, + 'deps.support.junit.jupiter': { + key: 'deps.support.junit.jupiter', + value: '5.0.1', + }, + 'deps.support.junit.platform': { + key: 'deps.support.junit.platform', + value: '1.0.1', + }, + 'deps.support.design': { + key: 'deps.support.design', + value: 'com.android.support:design:26.0.2', + }, + 'deps.picasso': { + key: 'deps.picasso', + value: 'com.squareup.picasso:picasso:2.5.2', + }, + }); }); it('map with interpolated dependency strings', () => {