diff --git a/.github/workflows/android-device-test.yml b/.github/workflows/android-device-test.yml index df0dceb4b8b..17c40465683 100644 --- a/.github/workflows/android-device-test.yml +++ b/.github/workflows/android-device-test.yml @@ -158,14 +158,8 @@ jobs: externalData: ${{ env.external_data_arn }} testSpecArn: ${{ matrix.test.testSpecArn }} - - name: Upload benchmark results to S3 - if: always() && env.run_device_test == 'true' && matrix.test.name == 'Android Benchmark' - run: | - npm install - node scripts/aws-device-farm/download-benchmark-results.mjs ${{ steps.aws_device_farm_run.outputs.runArn }} - - - name: Log Test Spec Output - if: failure() && env.run_device_test == 'true' + - name: Store Test Artifacts + if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true' run: | npm install temp_dir="$(mktemp -d)" @@ -173,7 +167,7 @@ jobs: zip -r test_artifacts.zip "$temp_dir" - name: Upload Test Artifacts - if: failure() && env.run_device_test == 'true' + if: (matrix.test.name == 'Android Benchmark' || failure()) && env.run_device_test == 'true' uses: actions/upload-artifact@v4 with: name: "Test Artifacts ${{ matrix.test.name }}" diff --git a/platform/android/MapLibreAndroidTestApp/src/main/AndroidManifest.xml b/platform/android/MapLibreAndroidTestApp/src/main/AndroidManifest.xml index 9be783948a1..f7793b89804 100644 --- a/platform/android/MapLibreAndroidTestApp/src/main/AndroidManifest.xml +++ b/platform/android/MapLibreAndroidTestApp/src/main/AndroidManifest.xml @@ -13,7 +13,9 @@ android:label="@string/app_name" android:roundIcon="@drawable/ic_launcher_round" android:supportsRtl="true" - android:theme="@style/AppTheme"> + android:theme="@style/AppTheme" + android:debuggable="true" + tools:ignore="HardcodedDebugMode"> , val styleURLs: List, - val resultsAPI: String = "" ) { init { if (styleNames.size != styleURLs.size) @@ -47,10 +42,6 @@ data class BenchmarkInputData( } } -/** - * Prepares JSON payload that is sent to the API that collects benchmark results. - * See https://github.com/maplibre/ci-runners - */ @SuppressLint("NewApi") fun jsonPayload(styleNames: List, fpsResults: BenchmarkResults, encodingTimeResults: BenchmarkResults, renderingTimeResults: BenchmarkResults): JsonObject { return buildJsonObject { @@ -141,14 +132,12 @@ class BenchmarkActivity : AppCompatActivity() { val jsonElement = Json.parseToJsonElement(jsonFileContents) val styleNames = jsonElement.jsonObject["styleNames"]?.jsonArray?.map { it.jsonPrimitive.content } val styleURLs = jsonElement.jsonObject["styleURLs"]?.jsonArray?.map { it.jsonPrimitive.content } - val resultsAPI = jsonElement.jsonObject["resultsAPI"]?.jsonPrimitive?.content - if (styleNames == null || styleURLs == null || resultsAPI == null) { + if (styleNames == null || styleURLs == null) { throw Error("${jsonFile.name} is missing elements") } return BenchmarkInputData( styleNames = styleNames.toList(), - styleURLs = styleURLs.toList(), - resultsAPI = resultsAPI + styleURLs = styleURLs.toList() ) } else { Logger.i(TAG, "${jsonFile.name} not found, reading from developer-config.xml") @@ -166,8 +155,20 @@ class BenchmarkActivity : AppCompatActivity() { // return default return BenchmarkInputData( - styleNames = listOf("MapLibre Demotiles"), - styleURLs = listOf("https://demotiles.maplibre.org/style.json") + styleNames = listOf( + "AWS Open Data Standard Light", + "Facebook Light", + "Americana", + "Protomaps Light", + "Versatiles Colorful" + ), + styleURLs = listOf( + "https://maps.geo.us-east-2.amazonaws.com/maps/v0/maps/OpenDataStyle/style-descriptor?key=v1.public.eyJqdGkiOiI1NjY5ZTU4My0yNWQwLTQ5MjctODhkMS03OGUxOTY4Y2RhMzgifR_7GLT66TNRXhZJ4KyJ-GK1TPYD9DaWuc5o6YyVmlikVwMaLvEs_iqkCIydspe_vjmgUVsIQstkGoInXV_nd5CcmqRMMa-_wb66SxDdbeRDvmmkpy2Ow_LX9GJDgL2bbiCws0wupJPFDwWCWFLwpK9ICmzGvNcrPbX5uczOQL0N8V9iUvziA52a1WWkZucIf6MUViFRf3XoFkyAT15Ll0NDypAzY63Bnj8_zS8bOaCvJaQqcXM9lrbTusy8Ftq8cEbbK5aMFapXRjug7qcrzUiQ5sr0g23qdMvnKJQFfo7JuQn8vwAksxrQm6A0ByceEXSfyaBoVpFcTzEclxUomhY.NjAyMWJkZWUtMGMyOS00NmRkLThjZTMtODEyOTkzZTUyMTBi", + "https://external.xx.fbcdn.net/maps/vt/style/canterbury_1_0/?locale=en_US", + "https://americanamap.org/style.json", + "https://api.protomaps.com/styles/v2/light.json?key=e761cc7daedf832a", + "https://tiles.versatiles.org/assets/styles/colorful.json", + ) ) } @@ -313,28 +314,16 @@ class BenchmarkActivity : AppCompatActivity() { mapView.onLowMemory() } - private fun sendResults() { - val api = inputData.resultsAPI - if (api.isEmpty()) { - Logger.i(TAG, "Not sending results to API") - return - } - - val client = OkHttpClient() - + private fun storeResults() { val payload = jsonPayload(inputData.styleNames, fpsResults, encodingTimeResults, renderingTimeResults) - Logger.i(TAG, "Sending JSON payload to API: $payload") - - val request = Request.Builder() - .url(api) - .post( - Json.encodeToString(payload).toRequestBody("application/json".toMediaType())) - .build() - client.newCall(request).execute() + + val dataDir = this.filesDir + val benchmarkResultsFile = File(dataDir, "benchmark_results.json") + benchmarkResultsFile.writeText(Json.encodeToString(payload)) } private fun benchmarkDone() { - sendResults() + storeResults() setResult(Activity.RESULT_OK) finish() } diff --git a/platform/android/gradle.properties b/platform/android/gradle.properties index 0f4c01d1d5f..a274ee6911d 100644 --- a/platform/android/gradle.properties +++ b/platform/android/gradle.properties @@ -3,4 +3,5 @@ systemProp.org.gradle.internal.http.connectionTimeout=360000 systemProp.org.gradle.internal.http.socketTimeout=360000 org.gradle.jvmargs=-Xmx4096M android.nonTransitiveRClass=false -android.nonFinalResIds=false \ No newline at end of file +android.nonFinalResIds=false +android.injected.androidTest.leaveApksInstalledAfterRun=true diff --git a/platform/android/settings.gradle b/platform/android/settings.gradle index 3fff1e3008b..79419e6965d 100644 --- a/platform/android/settings.gradle +++ b/platform/android/settings.gradle @@ -8,10 +8,10 @@ rootProject.name = "MapLibre Native for Android" def renderTestProjectDir = new File(rootDir, '../../render-test/android') includeBuild(renderTestProjectDir) { - name ="Render Test App" + name ="renderTestApp" } def cppTestProjectDir = new File(rootDir, '../../test/android') includeBuild(cppTestProjectDir) { - name ="C++ Unit Test App" + name ="cppUnitTestsApp" } \ No newline at end of file diff --git a/scripts/aws-device-farm/download-benchmark-results.mjs b/scripts/aws-device-farm/download-benchmark-results.mjs deleted file mode 100644 index e025ba32c27..00000000000 --- a/scripts/aws-device-farm/download-benchmark-results.mjs +++ /dev/null @@ -1,129 +0,0 @@ -// This script parses logcat output from AWS Device Farm to extract benchmark results. -// It also uploads the results to S3. -// android-benchmark-render/{gitRevision}/{deviceModel}.json - -import https from "node:https"; -import readline from "node:readline"; - -import { - ListJobsCommand, - ListArtifactsCommand, -} from "@aws-sdk/client-device-farm"; -import { PutObjectCommand } from "@aws-sdk/client-s3"; -import { getDeviceFarmClient } from "./device-farm-client.mjs"; -import { getS3Client } from "./s3-client.mjs"; - -/** - * Retrieves a log file and looks for the line that includes the benchmark results. - * Resolves to the log line or an empty string if the log line was not found. - * - * @param {string} artifactUrl - * @returns {Promise} - */ -function getBenchmarkResult(artifactUrl) { - return new Promise((resolve, reject) => { - https.get(artifactUrl, (res) => { - const rl = readline.createInterface({ - input: res, - }); - - rl.on("line", (line) => { - if (line.includes('Benchmark {"resultsPerStyle"')) { - resolve(line); - rl.close(); - } - }); - - rl.on("close", () => { - resolve(""); - }); - }); - }); -} - -function usage() { - console.error("Downloads benchmark results from AWS Device Farm"); - console.error(`Usage: node ${process.argv.at(1)} RUN_ARN`); - process.exit(1); -} - -const deviceFarmClient = getDeviceFarmClient(); - -if (process.argv.length !== 3) usage(); - -const arn = process.argv.at(2); - -const listJobsCommand = new ListJobsCommand({ arn }); -const { jobs } = await deviceFarmClient.send(listJobsCommand); - -if (!jobs) throw new Error("Failed to retrieve jobs"); - -const passedJobs = jobs.filter((job) => job.result === "PASSED" && job.arn); -const failedJobs = jobs.filter((job) => job.result !== "PASSED" || !job.arn); - -failedJobs.forEach((job) => { - console.error( - `job.result not PASSED. Skipping.\n$${JSON.stringify(job, null, 0)}` - ); -}); - -// retrieve all artifacts that are files (logcat logs are file artifacts) -const artifacts = await Promise.all( - passedJobs.map((job) => - deviceFarmClient.send( - new ListArtifactsCommand({ - arn: job.arn, - type: "FILE", - }) - ) - ) -); - -// only consider artifacts that are logcat logs -const logcatArtifacts = artifacts.flatMap((listArtifactsOutput) => - listArtifactsOutput.artifacts - ? listArtifactsOutput.artifacts.filter( - (artifact) => - artifact.type === "DEVICE_LOG" && artifact.name === "Logcat" - ) - : [] -); - -const benchmarkResultLogLines = await Promise.all( - logcatArtifacts.flatMap((artifact) => artifact.url ? [getBenchmarkResult(artifact.url)] : []) -); - -// parse the log lines to objects -/** @type {{gitRevision?: string, model?: string}[]} */ -const benchmarkResults = benchmarkResultLogLines - .map((line) => { - const firstCurly = line.indexOf("{"); - if (firstCurly === -1) return ""; - return line.substring(firstCurly); - }) - .filter((line) => !!line) - .map((jsonVal) => JSON.parse(jsonVal)); - -const s3Client = getS3Client(); - -// store benchmark results as JSON files on S3 -await Promise.all( - benchmarkResults.flatMap((result) => { - if (typeof result.model !== "string") { - console.error("Result missing model"); - return []; - } - if (typeof result.gitRevision !== "string") { - console.error("Result missing gitRevision"); - return []; - } - - const command = new PutObjectCommand({ - Bucket: "maplibre-native", // use environment variable - Key: `android-benchmark-render/${result.gitRevision}/${result.model}.json`, - ContentType: "application/json", - Body: JSON.stringify(result, null, 2), - }); - return [s3Client.send(command)]; - }) -);