diff --git a/buildSrc/src/main/kotlin/com/datadog/gradle/config/FlavorBuildConfig.kt b/buildSrc/src/main/kotlin/com/datadog/gradle/config/FlavorBuildConfig.kt index 108249abcf..cb1ab1d199 100644 --- a/buildSrc/src/main/kotlin/com/datadog/gradle/config/FlavorBuildConfig.kt +++ b/buildSrc/src/main/kotlin/com/datadog/gradle/config/FlavorBuildConfig.kt @@ -41,6 +41,11 @@ fun configureFlavorForSampleApp(flavor: ApplicationProductFlavor, rootDir: File) "DD_OVERRIDE_RUM_URL", "\"${config.rumEndpoint}\"" ) + flavor.buildConfigField( + "String", + "DD_OVERRIDE_SESSION_REPLAY_URL", + "\"${config.sessionReplayEndpoint}\"" + ) flavor.buildConfigField( "String", "DD_RUM_APPLICATION_ID", diff --git a/buildSrc/src/main/kotlin/com/datadog/gradle/config/SampleAppConfig.kt b/buildSrc/src/main/kotlin/com/datadog/gradle/config/SampleAppConfig.kt index 0dc34fd696..bdad42212e 100644 --- a/buildSrc/src/main/kotlin/com/datadog/gradle/config/SampleAppConfig.kt +++ b/buildSrc/src/main/kotlin/com/datadog/gradle/config/SampleAppConfig.kt @@ -10,6 +10,7 @@ data class SampleAppConfig( val logsEndpoint: String = "", val tracesEndpoint: String = "", val rumEndpoint: String = "", + val sessionReplayEndpoint: String = "", val token: String = "", val rumApplicationId: String = "", val apiKey: String = "", diff --git a/dd-sdk-android/src/main/kotlin/com/datadog/android/core/internal/net/DataOkHttpUploaderV2.kt b/dd-sdk-android/src/main/kotlin/com/datadog/android/core/internal/net/DataOkHttpUploaderV2.kt index 93e2b5f281..d02ffda4b7 100644 --- a/dd-sdk-android/src/main/kotlin/com/datadog/android/core/internal/net/DataOkHttpUploaderV2.kt +++ b/dd-sdk-android/src/main/kotlin/com/datadog/android/core/internal/net/DataOkHttpUploaderV2.kt @@ -29,11 +29,10 @@ internal abstract class DataOkHttpUploaderV2( internal enum class TrackType(val trackName: String) { LOGS("logs"), RUM("rum"), - SPANS("spans"), - SESSION_REPLAY("replay") + SPANS("spans") } - internal val uploaderName = javaClass.simpleName + private val uploaderName = javaClass.simpleName internal val clientToken = if (isValidHeaderValue(rawClientToken)) rawClientToken else "" internal val source: String = sanitizeHeaderValue(rawSource) @@ -113,7 +112,7 @@ internal abstract class DataOkHttpUploaderV2( return builder.build() } - internal fun buildUrl(): String { + private fun buildUrl(): String { val queryParams = buildQueryParameters() return if (queryParams.isEmpty()) { intakeUrl @@ -122,7 +121,7 @@ internal abstract class DataOkHttpUploaderV2( } } - internal fun buildHeaders(builder: Request.Builder, requestId: String) { + private fun buildHeaders(builder: Request.Builder, requestId: String) { builder.addHeader(HEADER_API_KEY, clientToken) builder.addHeader(HEADER_EVP_ORIGIN, source) builder.addHeader(HEADER_EVP_ORIGIN_VERSION, sdkVersion) @@ -135,7 +134,7 @@ internal abstract class DataOkHttpUploaderV2( return emptyMap() } - internal fun responseCodeToUploadStatus(code: Int): UploadStatus { + private fun responseCodeToUploadStatus(code: Int): UploadStatus { return when (code) { HTTP_ACCEPTED -> UploadStatus.SUCCESS HTTP_BAD_REQUEST -> UploadStatus.HTTP_CLIENT_ERROR @@ -192,7 +191,6 @@ internal abstract class DataOkHttpUploaderV2( internal const val CONTENT_TYPE_JSON = "application/json" internal const val CONTENT_TYPE_TEXT_UTF8 = "text/plain;charset=UTF-8" - internal const val CONTENT_TYPE_MUTLIPART_FORM = "multipart/form-data" private const val UPLOAD_URL = "%s/api/v2/%s" diff --git a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/SessionReplayFeature.kt b/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/SessionReplayFeature.kt index 89f59d9bfa..e889041a60 100644 --- a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/SessionReplayFeature.kt +++ b/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/SessionReplayFeature.kt @@ -79,12 +79,7 @@ internal class SessionReplayFeature( return SessionReplayRequestFactory( SessionReplayOkHttpUploader( configuration.endpointUrl, - coreFeature.clientToken, - coreFeature.sourceName, - coreFeature.sdkVersion, - coreFeature.okHttpClient, - coreFeature.androidInfoProvider, - coreFeature + coreFeature.okHttpClient ) ) } diff --git a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/domain/SessionReplayRequestFactory.kt b/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/domain/SessionReplayRequestFactory.kt index 6eaa723d74..7fd45cada1 100644 --- a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/domain/SessionReplayRequestFactory.kt +++ b/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/domain/SessionReplayRequestFactory.kt @@ -7,8 +7,8 @@ package com.datadog.android.sessionreplay.internal.domain import com.datadog.android.core.internal.net.DataOkHttpUploaderV2 -import com.datadog.android.sessionreplay.internal.net.BatchesToSegmentsMapper import com.datadog.android.sessionreplay.internal.net.SessionReplayOkHttpUploader +import com.datadog.android.sessionreplay.net.BatchesToSegmentsMapper import com.datadog.android.v2.api.Request import com.datadog.android.v2.api.RequestFactory import com.datadog.android.v2.api.context.DatadogContext @@ -27,6 +27,7 @@ internal class SessionReplayRequestFactory( // Also add the necessary unit tests once this is done. batchToSegmentsMapper.map(batchData).forEach { sessionReplayOkHttpUploader.upload( + context, it.first, (it.second.toString() + "\n").toByteArray() ) @@ -34,10 +35,9 @@ internal class SessionReplayRequestFactory( return Request( sessionReplayOkHttpUploader.requestId, "", - sessionReplayOkHttpUploader.buildUrl(), + sessionReplayOkHttpUploader.buildUrl(context), mapOf( - DataOkHttpUploaderV2.HEADER_API_KEY - to sessionReplayOkHttpUploader.clientToken + DataOkHttpUploaderV2.HEADER_API_KEY to context.clientToken ), ByteArray(0) ) diff --git a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/SessionReplayOkHttpUploader.kt b/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/SessionReplayOkHttpUploader.kt index 8525a5a974..b705f2a127 100644 --- a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/SessionReplayOkHttpUploader.kt +++ b/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/SessionReplayOkHttpUploader.kt @@ -6,14 +6,16 @@ package com.datadog.android.sessionreplay.internal.net -import com.datadog.android.core.internal.CoreFeature -import com.datadog.android.core.internal.net.DataOkHttpUploaderV2 import com.datadog.android.core.internal.net.UploadStatus -import com.datadog.android.core.internal.system.AndroidInfoProvider import com.datadog.android.core.internal.utils.devLogger import com.datadog.android.core.internal.utils.sdkLogger +import com.datadog.android.log.Logger import com.datadog.android.rum.RumAttributes import com.datadog.android.sessionreplay.model.MobileSegment +import com.datadog.android.sessionreplay.net.BytesCompressor +import com.datadog.android.v2.api.context.DatadogContext +import java.util.Locale +import java.util.UUID import okhttp3.Call import okhttp3.MediaType import okhttp3.MultipartBody @@ -24,44 +26,64 @@ import okhttp3.RequestBody // instead from SessionReplayRequestFactory // This class is not test as it is meant for non - production usage. It will be dropped later. internal class SessionReplayOkHttpUploader( - endpoint: String, - clientToken: String, - source: String, - sdkVersion: String, - callFactory: Call.Factory, - androidInfoProvider: AndroidInfoProvider, - private val coreFeature: CoreFeature, + private val endpoint: String, + internal val callFactory: Call.Factory, + internal val internalLogger: Logger = sdkLogger, private val compressor: BytesCompressor = BytesCompressor() -) : DataOkHttpUploaderV2( - buildUrl(endpoint, TrackType.SESSION_REPLAY), - clientToken, - source, - sdkVersion, - callFactory, - CONTENT_TYPE_MUTLIPART_FORM, - androidInfoProvider, - sdkLogger ) { - private val tags: String - get() { - val elements = mutableListOf( - "${RumAttributes.SERVICE_NAME}:${coreFeature.serviceName}", - "${RumAttributes.APPLICATION_VERSION}:" + - coreFeature.packageVersionProvider.version, - "${RumAttributes.SDK_VERSION}:$sdkVersion", - "${RumAttributes.ENV}:${coreFeature.envName}" - ) - if (coreFeature.variant.isNotEmpty()) { - elements.add("${RumAttributes.VARIANT}:${coreFeature.variant}") + private val uploaderName = javaClass.simpleName + + internal val requestId: String by lazy { + UUID.randomUUID().toString() + } + + private fun userAgent(datadogContext: DatadogContext): String { + return sanitizeHeaderValue(System.getProperty(SYSTEM_UA)) + .ifBlank { + "Datadog/${sanitizeHeaderValue(datadogContext.sdkVersion)} " + + "(Linux; U; Android ${datadogContext.deviceInfo.osVersion}; " + + "${datadogContext.deviceInfo.deviceModel} " + + "Build/${datadogContext.deviceInfo.deviceBuildId})" } - return elements.joinToString(",") + } + + private val intakeUrl by lazy { + String.format( + Locale.US, + UPLOAD_URL, + endpoint, + "replay" + ) + } + + private fun tags(datadogContext: DatadogContext): String { + val elements = mutableListOf( + "${RumAttributes.SERVICE_NAME}:${datadogContext.service}", + "${RumAttributes.APPLICATION_VERSION}:" + + datadogContext.version, + "${RumAttributes.SDK_VERSION}:${datadogContext.sdkVersion}", + "${RumAttributes.ENV}:${datadogContext.env}" + ) + if (datadogContext.variant.isNotEmpty()) { + elements.add("${RumAttributes.VARIANT}:${datadogContext.variant}") } + return elements.joinToString(",") + } @Suppress("TooGenericExceptionCaught") - fun upload(mobileSegment: MobileSegment, mobileSegmentAsBinary: ByteArray): UploadStatus { + fun upload( + datadogContext: DatadogContext, + mobileSegment: MobileSegment, + mobileSegmentAsBinary: ByteArray + ): UploadStatus { val uploadStatus = try { - executeUploadRequest(mobileSegment, mobileSegmentAsBinary, requestId) + executeUploadRequest( + datadogContext, + mobileSegment, + mobileSegmentAsBinary, + requestId + ) } catch (e: Throwable) { internalLogger.e("Unable to upload batch data.", e) UploadStatus.NETWORK_ERROR @@ -87,16 +109,19 @@ internal class SessionReplayOkHttpUploader( return uploadStatus } + // region Internal + @Suppress("UnsafeThirdPartyFunctionCall") // Called within a try/catch block private fun executeUploadRequest( + datadogContext: DatadogContext, mobileSegment: MobileSegment, mobileSegmentAsBinary: ByteArray, requestId: String ): UploadStatus { - if (clientToken.isBlank()) { + if (datadogContext.clientToken.isBlank()) { return UploadStatus.INVALID_TOKEN_ERROR } - val request = buildRequest(mobileSegment, mobileSegmentAsBinary, requestId) + val request = buildRequest(datadogContext, mobileSegment, mobileSegmentAsBinary, requestId) val call = callFactory.newCall(request) val response = call.execute() response.close() @@ -105,15 +130,16 @@ internal class SessionReplayOkHttpUploader( @Suppress("UnsafeThirdPartyFunctionCall") // Called within a try/catch block private fun buildRequest( + datadogContext: DatadogContext, segment: MobileSegment, segmentAsBinary: ByteArray, requestId: String ): Request { val builder = Request.Builder() - .url(buildUrl()) + .url(intakeUrl) .post(buildRequestBody(segment, segmentAsBinary)) - buildHeaders(builder, requestId) + buildHeaders(datadogContext, builder, requestId) return builder.build() } @@ -131,6 +157,11 @@ internal class SessionReplayOkHttpUploader( compressedData ) ) + .addFormDataPart( + SEGMENT_FORM_KEY, + segment.session.id + + ) .addFormDataPart( APPLICATION_ID_FORM_KEY, segment.application.id @@ -170,17 +201,91 @@ internal class SessionReplayOkHttpUploader( .build() } - override fun buildQueryParameters(): Map { + internal fun buildUrl(datadogContext: DatadogContext): String { + val queryParams = buildQueryParameters(datadogContext) + return if (queryParams.isEmpty()) { + intakeUrl + } else { + intakeUrl + queryParams.map { "${it.key}=${it.value}" } + .joinToString("&", prefix = "?") + } + } + + private fun buildHeaders( + datadogContext: DatadogContext, + builder: Request.Builder, + requestId: String + ) { + builder.addHeader(HEADER_API_KEY, datadogContext.clientToken) + builder.addHeader(HEADER_EVP_ORIGIN, datadogContext.source) + builder.addHeader(HEADER_EVP_ORIGIN_VERSION, datadogContext.sdkVersion) + builder.addHeader(HEADER_USER_AGENT, userAgent(datadogContext)) + builder.addHeader(HEADER_CONTENT_TYPE, CONTENT_TYPE_MULTIPART_FORM) + builder.addHeader(HEADER_REQUEST_ID, requestId) + } + + private fun responseCodeToUploadStatus(code: Int): UploadStatus { + return when (code) { + HTTP_ACCEPTED -> UploadStatus.SUCCESS + HTTP_BAD_REQUEST -> UploadStatus.HTTP_CLIENT_ERROR + HTTP_UNAUTHORIZED -> UploadStatus.INVALID_TOKEN_ERROR + HTTP_FORBIDDEN -> UploadStatus.INVALID_TOKEN_ERROR + HTTP_CLIENT_TIMEOUT -> UploadStatus.HTTP_CLIENT_RATE_LIMITING + HTTP_ENTITY_TOO_LARGE -> UploadStatus.HTTP_CLIENT_ERROR + HTTP_TOO_MANY_REQUESTS -> UploadStatus.HTTP_CLIENT_RATE_LIMITING + HTTP_INTERNAL_ERROR -> UploadStatus.HTTP_SERVER_ERROR + HTTP_UNAVAILABLE -> UploadStatus.HTTP_SERVER_ERROR + else -> UploadStatus.UNKNOWN_ERROR + } + } + + private fun sanitizeHeaderValue(value: String?): String { + return value?.filter { isValidHeaderValueChar(it) }.orEmpty() + } + + private fun isValidHeaderValueChar(c: Char): Boolean { + return c == '\t' || c in '\u0020' until '\u007F' + } + + private fun buildQueryParameters(datadogContext: DatadogContext): Map { return mapOf( - QUERY_PARAM_SOURCE to source, - QUERY_PARAM_TAGS to tags, - QUERY_PARAM_EVP_ORIGIN_KEY to source, - QUERY_PARAM_EVP_ORIGIN_VERSION_KEY to sdkVersion + QUERY_PARAM_SOURCE to datadogContext.source, + QUERY_PARAM_TAGS to tags(datadogContext), + QUERY_PARAM_EVP_ORIGIN_KEY to datadogContext.source, + QUERY_PARAM_EVP_ORIGIN_VERSION_KEY to datadogContext.sdkVersion ) } + // endregion + companion object { + const val SYSTEM_UA = "http.agent" + + const val HTTP_ACCEPTED = 202 + + const val HTTP_BAD_REQUEST = 400 + const val HTTP_UNAUTHORIZED = 401 + const val HTTP_FORBIDDEN = 403 + const val HTTP_CLIENT_TIMEOUT = 408 + const val HTTP_ENTITY_TOO_LARGE = 413 + const val HTTP_TOO_MANY_REQUESTS = 429 + + const val HTTP_INTERNAL_ERROR = 500 + const val HTTP_UNAVAILABLE = 503 + + internal const val HEADER_API_KEY = "DD-API-KEY" + internal const val HEADER_EVP_ORIGIN = "DD-EVP-ORIGIN" + internal const val HEADER_EVP_ORIGIN_VERSION = "DD-EVP-ORIGIN-VERSION" + internal const val HEADER_REQUEST_ID = "DD-REQUEST-ID" + internal const val HEADER_CONTENT_TYPE = "Content-Type" + internal const val HEADER_USER_AGENT = "User-Agent" + + internal const val QUERY_PARAM_SOURCE = "ddsource" + internal const val QUERY_PARAM_TAGS = "ddtags" + + private const val UPLOAD_URL = "%s/api/v2/%s" + internal const val QUERY_PARAM_EVP_ORIGIN_VERSION_KEY = "dd-evp-origin-version" internal const val QUERY_PARAM_EVP_ORIGIN_KEY = "dd-evp-origin" internal const val APPLICATION_ID_FORM_KEY = "application.id" @@ -194,5 +299,8 @@ internal class SessionReplayOkHttpUploader( internal const val SOURCE_FORM_KEY = "end" internal const val SEGMENT_FORM_KEY = "segment" internal const val CONTENT_TYPE_BINARY = "application/octet-stream" + internal const val CONTENT_TYPE_MULTIPART_FORM = "multipart/form-data" + private const val HEADER_ENCODING = "Content-Encoding" + private const val ENCODING_GZIP = "gzip" } } diff --git a/dd-sdk-android/src/test/kotlin/com/datadog/android/core/internal/net/GzipRequestInterceptorTest.kt b/dd-sdk-android/src/test/kotlin/com/datadog/android/core/internal/net/GzipRequestInterceptorTest.kt index 5cd634cc02..d991bbe605 100644 --- a/dd-sdk-android/src/test/kotlin/com/datadog/android/core/internal/net/GzipRequestInterceptorTest.kt +++ b/dd-sdk-android/src/test/kotlin/com/datadog/android/core/internal/net/GzipRequestInterceptorTest.kt @@ -111,11 +111,19 @@ internal class GzipRequestInterceptorTest { } @Test - fun `M keep original body W intercept { MultipartBody }`() { + fun `M keep original body W intercept { MultipartBody }`(forge: Forge) { // Given + val fakeMultipartBody = MultipartBody + .Builder() + .addFormDataPart( + forge.aString(), + forge.aString(), + RequestBody.create(null, fakeBody.toByteArray()) + ) + .build() + fakeRequest = fakeRequest.newBuilder() - .header("Content-Encoding", "identity") - .post(MultipartBody.create(null, fakeBody)) + .post(fakeMultipartBody) .build() fakeResponse = forgeResponse() stubChain() @@ -128,13 +136,12 @@ internal class GzipRequestInterceptorTest { verify(mockChain).proceed(capture()) val buffer = Buffer() val stream = ByteArrayOutputStream() - lastValue.body()!!.writeTo(buffer) + val part = (lastValue.body() as MultipartBody).part(0) + part.body().writeTo(buffer) buffer.copyTo(stream) assertThat(stream.toString()) .isEqualTo(fakeBody) - assertThat(lastValue.header("Content-Encoding")) - .isEqualTo("identity") } assertThat(response) .isSameAs(fakeResponse) diff --git a/detekt.yml b/detekt.yml index b523ca58ef..7e020493c4 100644 --- a/detekt.yml +++ b/detekt.yml @@ -607,6 +607,12 @@ datadog: - "android.content.res.Resources.openRawResource(kotlin.Int):android.content.res.Resources.NotFoundException" # endregion # region Java File + - "java.io.ByteArrayOutputStream.write(kotlin.ByteArray, kotlin.Int, kotlin.Int):java.lang.IndexOutOfBoundsException" + - "java.nio.ByteBuffer.allocate(kotlin.Int):java.lang.IllegalArgumentException" + - "java.nio.ByteBuffer.array():java.nio.ReadOnlyBufferException,java.lang.UnsupportedOperationException" + - "java.nio.ByteBuffer.put(kotlin.ByteArray):java.nio.BufferOverflowException,java.nio.ReadOnlyBufferException" + - "java.nio.ByteBuffer.putInt(kotlin.Int):java.nio.BufferOverflowException,java.nio.ReadOnlyBufferException" + - "java.nio.ByteBuffer.putShort(kotlin.Short):java.nio.BufferOverflowException,java.nio.ReadOnlyBufferException" - "java.io.File.inputStream():java.io.FileNotFoundException,java.lang.SecurityException" - "java.io.File.canRead():java.lang.SecurityException" - "java.io.File.canWrite():java.lang.SecurityException" @@ -632,12 +638,6 @@ datadog: - "java.io.InputStream.read(kotlin.ByteArray, kotlin.Int, kotlin.Int):java.io.IOException" - "java.io.InputStream.reset():java.io.IOException" - "java.io.InputStream.skip(kotlin.Long):java.io.IOException" - - "java.io.ByteArrayOutputStream.write(kotlin.ByteArray, kotlin.Int, kotlin.Int):java.lang.IndexOutOfBoundsException" - - "java.nio.ByteBuffer.allocate(kotlin.Int):java.lang.IllegalArgumentException" - - "java.nio.ByteBuffer.array():java.nio.ReadOnlyBufferException,java.lang.UnsupportedOperationException" - - "java.nio.ByteBuffer.put(kotlin.ByteArray):java.nio.BufferOverflowException,java.nio.ReadOnlyBufferException" - - "java.nio.ByteBuffer.putInt(kotlin.Int):java.nio.BufferOverflowException,java.nio.ReadOnlyBufferException" - - "java.nio.ByteBuffer.putShort(kotlin.Short):java.nio.BufferOverflowException,java.nio.ReadOnlyBufferException" - "java.nio.channels.FileChannel.lock():java.io.IOException,java.lang.IllegalStateException" - "java.nio.channels.FileLock.release():java.io.IOException" # endregion @@ -970,6 +970,7 @@ datadog: - "java.io.StringWriter.constructor()" - "java.io.ByteArrayOutputStream.constructor(kotlin.Int)" - "java.io.ByteArrayOutputStream.toByteArray()" + - "java.io.ByteArrayOutputStream.use(kotlin.Function1)" # endregion # region Java misc - "java.lang.Class.hashCode()" @@ -1062,6 +1063,7 @@ datadog: - "kotlin.collections.List.orEmpty()" - "kotlin.collections.List.reversed()" - "kotlin.collections.List.shuffled()" + - "kotlin.collections.List.sortedBy(kotlin.Function1)" - "kotlin.collections.List.subList(kotlin.Int, kotlin.Int)" - "kotlin.collections.List.sumOf(kotlin.Function1)" - "kotlin.collections.List.take(kotlin.Int)" diff --git a/library/dd-sdk-android-session-replay/apiSurface b/library/dd-sdk-android-session-replay/apiSurface index ac84ca0b3f..440163cc4a 100644 --- a/library/dd-sdk-android-session-replay/apiSurface +++ b/library/dd-sdk-android-session-replay/apiSurface @@ -225,6 +225,12 @@ data class com.datadog.android.sessionreplay.model.MobileSegment fun toJson(): com.google.gson.JsonElement companion object fun fromJson(kotlin.String): Vertical +class com.datadog.android.sessionreplay.net.BatchesToSegmentsMapper + fun map(List): List> + companion object +class com.datadog.android.sessionreplay.net.BytesCompressor + fun compressBytes(ByteArray): ByteArray + companion object data class com.datadog.android.sessionreplay.processor.EnrichedRecord constructor(String, String, String, List) fun toJson(): String diff --git a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/BatchesToSegmentsMapper.kt b/library/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/net/BatchesToSegmentsMapper.kt similarity index 53% rename from dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/BatchesToSegmentsMapper.kt rename to library/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/net/BatchesToSegmentsMapper.kt index b677fb2ca4..2de988d800 100644 --- a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/BatchesToSegmentsMapper.kt +++ b/library/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/net/BatchesToSegmentsMapper.kt @@ -4,7 +4,7 @@ * Copyright 2016-Present Datadog, Inc. */ -package com.datadog.android.sessionreplay.internal.net +package com.datadog.android.sessionreplay.net import com.datadog.android.sessionreplay.model.MobileSegment import com.datadog.android.sessionreplay.processor.EnrichedRecord @@ -14,8 +14,12 @@ import com.google.gson.JsonObject import com.google.gson.JsonParseException import com.google.gson.JsonParser -internal class BatchesToSegmentsMapper { +/** Maps a batch to a List> for uploading. + * This class is meant for internal usage. + */ +class BatchesToSegmentsMapper { + @Suppress("UndocumentedPublicFunction") fun map(batchData: List): List> { return groupBatchDataIntoSegments(batchData) } @@ -24,44 +28,6 @@ internal class BatchesToSegmentsMapper { private fun groupBatchDataIntoSegments(batchData: List): List> { - return groupBatchDataByRumContext(batchData) - .filter { !it.value.isEmpty } - .map { entry -> - val records = entry.value.map { it.asJsonObject } - - // we are filtering out empty records so we are safe to call first/last functions - @Suppress("UnsafeThirdPartyFunctionCall") - val startTimestamp = records.first().getAsJsonPrimitive(TIMESTAMP_KEY).asLong - - @Suppress("UnsafeThirdPartyFunctionCall") - val stopTimestamp = records.last().getAsJsonPrimitive(TIMESTAMP_KEY).asLong - val hasFullSnapshotRecord = hasFullSnapshotRecord(records) - val segment = MobileSegment( - MobileSegment.Application(entry.key.applicationId), - MobileSegment.Session(entry.key.sessionId), - MobileSegment.View(entry.key.viewId), - startTimestamp, - stopTimestamp, - records.size.toLong(), - // TODO: RUMM-2518 Find a way or alternative to provide a reliable indexInView - null, - hasFullSnapshotRecord, - MobileSegment.Source.ANDROID, - emptyList() - ) - val segmentAsJsonObject = segment.toJson().asJsonObject - segmentAsJsonObject.add(RECORDS_KEY, entry.value) - Pair(segment, segmentAsJsonObject) - } - } - - private fun hasFullSnapshotRecord(records: List) = - records.firstOrNull { - it.getAsJsonPrimitive(RECORD_TYPE_KEY).asLong == FULL_SNAPSHOT_RECORD_TYPE - } != null - - private fun groupBatchDataByRumContext(batchData: List): - Map { return batchData .mapNotNull { @Suppress("SwallowedException") @@ -90,8 +56,59 @@ internal class BatchesToSegmentsMapper { acc } } + .filter { !it.value.isEmpty } + .mapNotNull { entry -> + @Suppress("SwallowedException") + try { + groupToSegmentsPair(entry) + } catch (e: JsonParseException) { + // TODO: RUMM-2397 Add the proper logs here once the sdkLogger will be added + null + } catch (e: IllegalStateException) { + // TODO: RUMM-2397 Add the proper logs here once the sdkLogger will be added + null + } + } + } + + private fun groupToSegmentsPair(entry: Map.Entry): + Pair { + val records = entry.value + .map { it.asJsonObject } + .sortedBy { + it.getAsJsonPrimitive(TIMESTAMP_KEY).asLong + } + + // we are filtering out empty records so we are safe to call first/last functions + @Suppress("UnsafeThirdPartyFunctionCall") + val startTimestamp = records.first().getAsJsonPrimitive(TIMESTAMP_KEY).asLong + + @Suppress("UnsafeThirdPartyFunctionCall") + val stopTimestamp = records.last().getAsJsonPrimitive(TIMESTAMP_KEY).asLong + val hasFullSnapshotRecord = hasFullSnapshotRecord(records) + val segment = MobileSegment( + application = MobileSegment.Application(entry.key.applicationId), + session = MobileSegment.Session(entry.key.sessionId), + view = MobileSegment.View(entry.key.viewId), + start = startTimestamp, + end = stopTimestamp, + recordsCount = records.size.toLong(), + // TODO: RUMM-2518 Find a way or alternative to provide a reliable indexInView + indexInView = null, + hasFullSnapshot = hasFullSnapshotRecord, + source = MobileSegment.Source.ANDROID, + records = emptyList() + ) + val segmentAsJsonObject = segment.toJson().asJsonObject + segmentAsJsonObject.add(RECORDS_KEY, entry.value) + return Pair(segment, segmentAsJsonObject) } + private fun hasFullSnapshotRecord(records: List) = + records.firstOrNull { + it.getAsJsonPrimitive(RECORD_TYPE_KEY).asLong == FULL_SNAPSHOT_RECORD_TYPE + } != null + // endregion companion object { diff --git a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/BytesCompressor.kt b/library/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/net/BytesCompressor.kt similarity index 80% rename from dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/BytesCompressor.kt rename to library/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/net/BytesCompressor.kt index 3cf93501c1..e81af5e1e7 100644 --- a/dd-sdk-android/src/main/kotlin/com/datadog/android/sessionreplay/internal/net/BytesCompressor.kt +++ b/library/dd-sdk-android-session-replay/src/main/kotlin/com/datadog/android/sessionreplay/net/BytesCompressor.kt @@ -4,13 +4,17 @@ * Copyright 2016-Present Datadog, Inc. */ -package com.datadog.android.sessionreplay.internal.net +package com.datadog.android.sessionreplay.net import java.io.ByteArrayOutputStream import java.util.zip.Deflater -internal class BytesCompressor { +/** Compresses the payload data using the ZIP compression algorithm. + * This class is meant for internal usage. + */ +class BytesCompressor { + @Suppress("UndocumentedPublicFunction") fun compressBytes(uncompressedData: ByteArray): ByteArray { // Create the compressor with highest level of compression val deflater = Deflater(6) @@ -20,13 +24,16 @@ internal class BytesCompressor { // in order to align with dogweb way of decompressing the segments we need to compress // using the SYNC_FLUSH flag which adds the 0000FFFF flag at the end of the // compressed data - compress(deflater, uncompressedData, outputStream, Deflater.SYNC_FLUSH) - // in order to align with dogweb way of decompressing the segments we need to add - // a fake checksum at the end - compress(deflater, ByteArray(0), outputStream, Deflater.FULL_FLUSH) - deflater.end() + val compressedData = outputStream.use { + compress(deflater, uncompressedData, it, Deflater.SYNC_FLUSH) + // in order to align with dogweb way of decompressing the segments we need to add + // a fake checksum at the end + compress(deflater, ByteArray(0), it, Deflater.FULL_FLUSH) + deflater.end() + it.toByteArray() + } // Get the compressed data - return outputStream.toByteArray() + return compressedData } private fun compress( diff --git a/dd-sdk-android/src/test/kotlin/com/datadog/android/sessionreplay/internal/net/BatchesToSegmentMapperTest.kt b/library/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/net/BatchesToSegmentMapperTest.kt similarity index 81% rename from dd-sdk-android/src/test/kotlin/com/datadog/android/sessionreplay/internal/net/BatchesToSegmentMapperTest.kt rename to library/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/net/BatchesToSegmentMapperTest.kt index 45e34a0352..510e0b1cf1 100644 --- a/dd-sdk-android/src/test/kotlin/com/datadog/android/sessionreplay/internal/net/BatchesToSegmentMapperTest.kt +++ b/library/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/net/BatchesToSegmentMapperTest.kt @@ -4,11 +4,12 @@ * Copyright 2016-Present Datadog, Inc. */ -package com.datadog.android.sessionreplay.internal.net +package com.datadog.android.sessionreplay.net import com.datadog.android.sessionreplay.model.MobileSegment import com.datadog.android.sessionreplay.processor.EnrichedRecord -import com.datadog.android.utils.forge.Configurator +import com.datadog.android.sessionreplay.utils.ForgeConfigurator +import com.google.gson.JsonParser import fr.xgouchet.elmyr.Forge import fr.xgouchet.elmyr.junit5.ForgeConfiguration import fr.xgouchet.elmyr.junit5.ForgeExtension @@ -27,7 +28,7 @@ import org.mockito.quality.Strictness ExtendWith(ForgeExtension::class) ) @MockitoSettings(strictness = Strictness.LENIENT) -@ForgeConfiguration(Configurator::class) +@ForgeConfiguration(ForgeConfigurator::class) internal class BatchesToSegmentMapperTest { lateinit var testedMapper: BatchesToSegmentsMapper @@ -75,11 +76,12 @@ internal class BatchesToSegmentMapperTest { val expectedEnrichedRecords: LinkedList = LinkedList() val fakeEnrichedRecords: List = fakeRecords .chunked(chunkSize) + .map { it.sortedBy { record -> record.timestamp() } } .flatMap { val rootRecord = forge.getForgery().copy(records = it) expectedEnrichedRecords.add(rootRecord) val subChunkSize = it.size / 2 - if (subChunkSize> 0) { + if (subChunkSize > 0) { it.chunked(subChunkSize).map { records -> rootRecord.copy(records = records) } @@ -112,9 +114,13 @@ internal class BatchesToSegmentMapperTest { forge: Forge ) { // Given - val fakeEnrichedRecords: List = forge.aList(1) { - forge.getForgery() - } + val fakeEnrichedRecords: List = forge + .aList(size = 1) { + forge.getForgery() + } + .map { + it.copy(records = it.records.sortedBy { record -> record.timestamp() }) + } val fakeBatchData = fakeEnrichedRecords.map { it.toJson().toByteArray() } val expectedStartTimestamp = fakeEnrichedRecords[0].records.first().timestamp() @@ -130,13 +136,17 @@ internal class BatchesToSegmentMapperTest { } @Test - fun `M use the first record in the list as end timestamp W groupDataIntoSegments`( + fun `M use the last record in the list as end timestamp W groupDataIntoSegments`( forge: Forge ) { // Given - val fakeEnrichedRecords: List = forge.aList(1) { - forge.getForgery() - } + val fakeEnrichedRecords: List = forge + .aList(size = 1) { + forge.getForgery() + } + .map { + it.copy(records = it.records.sortedBy { record -> record.timestamp() }) + } val fakeBatchData = fakeEnrichedRecords.map { it.toJson().toByteArray() } val expectedEndTimestamp = fakeEnrichedRecords[0].records.last().timestamp() @@ -183,6 +193,33 @@ internal class BatchesToSegmentMapperTest { assertThat(mappedSegments).isEmpty() } + @Test + fun `M return empty list W groupDataIntoSegments { records with missing timestamp key }`( + forge: Forge + ) { + // Given + val fakeBatchData = forge + .aList(size = 1) { + forge.getForgery() + } + .map { JsonParser.parseString(it.toJson()).asJsonObject } + .map { + val records = it.get(EnrichedRecord.RECORDS_KEY) + .asJsonArray + records.forEach { record -> + record.asJsonObject.remove(BatchesToSegmentsMapper.TIMESTAMP_KEY) + } + it.add(EnrichedRecord.RECORDS_KEY, records) + } + .map { it.toString().toByteArray() } + + // When + val mappedSegments = testedMapper.map(fakeBatchData) + + // Then + assertThat(mappedSegments).isEmpty() + } + // region Internal private fun EnrichedRecord.toSegment(): MobileSegment { diff --git a/dd-sdk-android/src/test/kotlin/com/datadog/android/sessionreplay/internal/net/BytesCompressorTest.kt b/library/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/net/BytesCompressorTest.kt similarity index 93% rename from dd-sdk-android/src/test/kotlin/com/datadog/android/sessionreplay/internal/net/BytesCompressorTest.kt rename to library/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/net/BytesCompressorTest.kt index 466784d24a..b4e9fd06c7 100644 --- a/dd-sdk-android/src/test/kotlin/com/datadog/android/sessionreplay/internal/net/BytesCompressorTest.kt +++ b/library/dd-sdk-android-session-replay/src/test/kotlin/com/datadog/android/sessionreplay/net/BytesCompressorTest.kt @@ -4,10 +4,10 @@ * Copyright 2016-Present Datadog, Inc. */ -package com.datadog.android.sessionreplay.internal.net +package com.datadog.android.sessionreplay.net import com.datadog.android.sessionreplay.processor.EnrichedRecord -import com.datadog.android.utils.forge.Configurator +import com.datadog.android.sessionreplay.utils.ForgeConfigurator import fr.xgouchet.elmyr.annotation.Forgery import fr.xgouchet.elmyr.junit5.ForgeConfiguration import fr.xgouchet.elmyr.junit5.ForgeExtension @@ -26,7 +26,7 @@ import org.mockito.quality.Strictness ExtendWith(ForgeExtension::class) ) @MockitoSettings(strictness = Strictness.LENIENT) -@ForgeConfiguration(Configurator::class) +@ForgeConfiguration(ForgeConfigurator::class) internal class BytesCompressorTest { lateinit var testedBytesCompressor: BytesCompressor diff --git a/sample/kotlin/src/main/kotlin/com/datadog/android/sample/SampleApplication.kt b/sample/kotlin/src/main/kotlin/com/datadog/android/sample/SampleApplication.kt index 1e84fe94b9..986261c9ac 100644 --- a/sample/kotlin/src/main/kotlin/com/datadog/android/sample/SampleApplication.kt +++ b/sample/kotlin/src/main/kotlin/com/datadog/android/sample/SampleApplication.kt @@ -163,6 +163,9 @@ class SampleApplication : Application() { if (BuildConfig.DD_OVERRIDE_RUM_URL.isNotBlank()) { configBuilder.useCustomRumEndpoint(BuildConfig.DD_OVERRIDE_RUM_URL) } + if (BuildConfig.DD_OVERRIDE_SESSION_REPLAY_URL.isNotBlank()) { + configBuilder.useSessionReplayEndpoint(BuildConfig.DD_OVERRIDE_SESSION_REPLAY_URL) + } return configBuilder.build() }