[KxSerialization] Fixed access private custom serializer on property

When a custom serializer is specified on a type and this type is used in a property of another serializable class, then on the JVM this leads to an error accessing the custom serializer class - because it is private and located in another package.

Fixes Kotlin/kotlinx.serialization#2495

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/BaseIrGenerator.kt

@@ -11,6 +11,7 @@ import org.jetbrains.kotlin.backend.jvm.functionByName
 import org.jetbrains.kotlin.backend.jvm.ir.fileParent
 import org.jetbrains.kotlin.backend.jvm.ir.representativeUpperBound
 import org.jetbrains.kotlin.descriptors.ClassKind
+import org.jetbrains.kotlin.descriptors.DescriptorVisibilities
 import org.jetbrains.kotlin.ir.UNDEFINED_OFFSET
 import org.jetbrains.kotlin.ir.builders.*
 import org.jetbrains.kotlin.ir.declarations.*
@@ -280,6 +281,7 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
             compilerContext,
             property.type,
             property.genericIndex,
+            property.ir.parentClassOrNull,
             genericGetter
         )
         val (functionToCall, args: List<IrExpression>) = if (innerSerial != null) whenHaveSerializer(innerSerial, sti) else whenDoNot(sti)
@@ -346,7 +348,8 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
                 serializerClassSymbol,
                 compilerContext,
                 property.type,
-                genericIndex = property.genericIndex
+                genericIndex = property.genericIndex,
+                property.ir.parentClassOrNull,
             ) { it, _ ->
                 val ir = generator.localSerializersFieldsDescriptors[it]
                 irGetField(irGet(dispatchReceiverParameter), ir.backingField!!)
@@ -469,6 +472,7 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
             compilerContext,
             property.type,
             null,
+            serializableClass,
             null
         )
     }
@@ -489,22 +493,38 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
         pluginContext: SerializationPluginContext,
         kType: IrType,
         genericIndex: Int? = null,
-        genericGetter: ((Int, IrType) -> IrExpression)? = null
+        rootSerializableClass: IrClass? = null,
+        genericGetter: ((Int, IrType) -> IrExpression)? = null,
     ): IrExpression? {
         val nullableSerClass = compilerContext.referenceProperties(SerialEntityNames.wrapIntoNullableCallableId).single()
         if (serializerClassOriginal == null) {
             if (genericIndex == null) return null
             return genericGetter?.invoke(genericIndex, kType)
         }
         if (serializerClassOriginal.owner.kind == ClassKind.OBJECT) {
-            return irGetObject(serializerClassOriginal)
+            val serializerClass = serializerClassOriginal.owner
+
+            val samePackage =
+                serializerClass.classId != null && serializerClass.classId?.packageFqName == rootSerializableClass?.classId?.packageFqName
+
+            // rootSerializableClass is only if the serializer is obtained for serializer getter
+            //   In this case, the private serializer will always be located in the same package, otherwise a syntax error will occur.
+            return if (rootSerializableClass == null || serializerClass.visibility != DescriptorVisibilities.PRIVATE || samePackage) {
+                // we can access the serializer object directly only if it is not private, or is located in the same package as the class using it
+                irGetObject(serializerClassOriginal)
+            } else {
+                val simpleType = (kType as? IrSimpleType) ?: error("Don't know how to work with type ${kType::class}")
+                callSerializerFromCompanion(simpleType, emptyList(), emptyList(), serializerClassOriginal.owner.classId)
+                    ?: error("Can't get serializer from companion's 'serializer()' function for type ${kType::class}")
+            }
         }
         fun instantiate(serializer: IrClassSymbol?, type: IrType): IrExpression? {
             val expr = serializerInstance(
                 serializer,
                 pluginContext,
                 type,
                 type.genericIndex,
+                rootSerializableClass,
                 genericGetter
             ) ?: return null
             return wrapWithNullableSerializerIfNeeded(type, expr, nullableSerClass)
@@ -593,7 +613,8 @@ abstract class BaseIrGenerator(private val currentClass: IrClass, final override
                                     serializer,
                                     pluginContext,
                                     type,
-                                    type.genericIndex
+                                    type.genericIndex,
+                                    rootSerializableClass
                                 ) { _, genericType ->
                                     serializerInstance(
                                         pluginContext.referenceClass(polymorphicSerializerId),

plugins/kotlinx-serialization/kotlinx-serialization.backend/src/org/jetbrains/kotlinx/serialization/compiler/backend/ir/SerializableIrGenerator.kt

@@ -342,7 +342,8 @@ class SerializableIrGenerator(
                             serial,
                             compilerContext,
                             arg.typeOrNull!!,
-                            genericIdx
+                            genericIdx,
+                            irClass
                         ) { it, _ ->
                             irGet(writeSelfFunction.valueParameters[3 + it])
                         }!!

plugins/kotlinx-serialization/testData/boxIr/privateCustomSerializer.kt

@@ -0,0 +1,47 @@
+// TARGET_BACKEND: JVM_IR
+
+// WITH_STDLIB
+
+// FILE: serializer.kt
+
+package a
+
+import kotlinx.serialization.*
+import kotlinx.serialization.encoding.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.json.*
+import kotlin.reflect.KClass
+import kotlin.test.*
+
+@Serializable(DataSerializer::class)
+data class Data(
+    val i: Int
+)
+
+
+
+@Serializer(forClass = Data::class)
+private object DataSerializer
+
+// FILE: holder.kt
+
+package b
+
+import kotlinx.serialization.*
+import kotlinx.serialization.encoding.*
+import kotlinx.serialization.descriptors.*
+import kotlinx.serialization.json.*
+import kotlin.reflect.KClass
+import kotlin.test.*
+import a.Data
+
+@Serializable
+data class Holder(
+    val data: Data
+)
+
+
+fun box(): String {
+    val json = Json.encodeToString(Holder(Data(1)))
+    return if (json == "{\"data\":{\"i\":1}}") "OK" else json
+}