Handle checkcast instruction in JvmValueTransferRelation

Summary: This introduces support for `checkcast` in the value transfer relation.

base/src/main/java/proguard/analysis/cpa/jvm/domain/value/JvmValueTransferRelation.java

@@ -10,6 +10,7 @@
 import java.util.Deque;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Objects;
 import proguard.analysis.cpa.defaults.StackAbstractState;
 import proguard.analysis.cpa.interfaces.AbstractState;
 import proguard.analysis.cpa.interfaces.Precision;
@@ -26,6 +27,7 @@
 import proguard.evaluation.ExecutingInvocationUnit;
 import proguard.evaluation.value.IdentifiedReferenceValue;
 import proguard.evaluation.value.TopValue;
+import proguard.evaluation.value.TypedReferenceValue;
 import proguard.evaluation.value.Value;
 import proguard.evaluation.value.ValueFactory;
 import proguard.evaluation.value.object.AnalyzedObjectFactory;
@@ -166,6 +168,38 @@ public void invokeMethod(
     super.invokeMethod(state, call, operands);
   }
 
+  @Override
+  protected ValueAbstractState handleCheckCast(ValueAbstractState state, String internalName) {
+    Value value = state.getValue();
+
+    // JVM spec demands that the operand of checkcast is a reference value, and we can't make
+    // a decision without type information. Note that this excludes arrays, as they are currently
+    // generally not supported in value analysis.
+    if (!(value instanceof TypedReferenceValue)) {
+      return getAbstractDefault();
+    }
+
+    // By the JVM spec, if objectref is null, the stack is left unchanged
+    TypedReferenceValue typedValue = (TypedReferenceValue) value;
+    if (typedValue.getValue().isNull()) {
+      return state;
+    }
+
+    boolean castSuccess;
+    Clazz referencedClass = typedValue.getReferencedClass();
+    if (referencedClass == null) {
+      // Fallback to type string comparison
+      String typeName = ClassUtil.internalTypeFromClassName(internalName);
+      castSuccess = Objects.equals(typeName, typedValue.getType());
+    } else {
+      castSuccess = referencedClass.extendsOrImplements(internalName);
+    }
+
+    return castSuccess ? state : getAbstractDefault();
+  }
+
+  // Private methods
+
   private void executeMethod(
       ConcreteCall call,
       JvmAbstractState<ValueAbstractState> state,

base/src/main/java/proguard/analysis/cpa/jvm/transfer/JvmTransferRelation.java

@@ -249,6 +249,12 @@ protected StateT isInstanceOf(StateT state, String type) {
     return getAbstractDefault();
   }
 
+  /** Returns an abstract state representing the result of the {@code checkcast} operation. */
+  protected StateT handleCheckCast(StateT state, String typeName) {
+    // Default behavior: ignore the instruction, checkcast leaves the object reference where it is
+    return state;
+  }
+
   /**
    * This {@link InstructionVisitor} performs generic operations (e.g., loads, stores) parametrized
    * by the specific behavior of {@link JvmTransferRelation} for instruction applications, method
@@ -642,6 +648,8 @@ public void visitConstantInstruction(
             break;
           }
         case Instruction.OP_CHECKCAST:
+          clazz.constantPoolEntryAccept(constantInstruction.constantIndex, constantLookupVisitor);
+          abstractState.push(handleCheckCast(abstractState.pop(), constantLookupVisitor.result));
           break;
         default: // should never happen
           throw new InvalidParameterException(

base/src/main/java/proguard/evaluation/value/ParticularReferenceValue.java

@@ -64,7 +64,7 @@ public Object value() {
   }
 
   @Override
-  public AnalyzedObject getValue() {
+  public @NotNull AnalyzedObject getValue() {
     return objectValue;
   }
 

base/src/main/java/proguard/evaluation/value/ReferenceValue.java

@@ -17,6 +17,7 @@
  */
 package proguard.evaluation.value;
 
+import org.jetbrains.annotations.NotNull;
 import proguard.classfile.Clazz;
 import proguard.evaluation.value.object.AnalyzedObject;
 import proguard.evaluation.value.object.AnalyzedObjectFactory;
@@ -39,6 +40,7 @@ public Object value() {
     return null;
   }
 
+  @NotNull
   public AnalyzedObject getValue() {
     return AnalyzedObjectFactory.createNull();
   }

base/src/test/kotlin/proguard/analysis/cpa/jvm/domain/value/CpaValueTest.kt

@@ -3,16 +3,29 @@ package proguard.analysis.cpa.jvm.domain.value
 import io.kotest.core.spec.style.FreeSpec
 import io.kotest.matchers.shouldBe
 import io.kotest.matchers.types.shouldBeInstanceOf
+import io.kotest.matchers.types.shouldNotBeInstanceOf
+import proguard.analysis.cpa.bam.BamCache
 import proguard.analysis.cpa.jvm.cfa.JvmCfa
 import proguard.analysis.cpa.jvm.util.CfaUtil
 import proguard.classfile.MethodSignature
-import proguard.classfile.ProgramClass
-import proguard.classfile.Signature
 import proguard.evaluation.value.ParticularReferenceValue
 import proguard.testutils.AssemblerSource
 import proguard.testutils.ClassPoolBuilder
 import proguard.testutils.JavaSource
 
+fun runCpa(cfa: JvmCfa, mainSignature: MethodSignature): BamCache<MethodSignature> {
+    val bamCpaRun = JvmValueBamCpaRun.Builder(cfa, mainSignature).setReduceHeap(true).build()
+    bamCpaRun.execute()
+    return bamCpaRun.cpa.cache
+}
+
+fun getLastState(cache: BamCache<MethodSignature>, mainSignature: MethodSignature): JvmValueAbstractState {
+    return cache
+        .get(mainSignature)
+        .map { it.reachedSet.asCollection().maxBy { (it as JvmValueAbstractState).programLocation.offset } }
+        .single() as JvmValueAbstractState
+}
+
 class CpaValueTest : FreeSpec({
 
     "Given overloaded fields with different types" - {
@@ -44,18 +57,13 @@ class CpaValueTest : FreeSpec({
                 }
                 """.trimIndent(),
             ),
-
         )
-        var cfa: JvmCfa = CfaUtil.createInterproceduralCfa(programClassPool)
-        val clazz = programClassPool.getClass("Test") as ProgramClass
-        val mainSignature = Signature.of(clazz, clazz.findMethod("main", null)) as MethodSignature
-        val bamCpaRun = JvmValueBamCpaRun.Builder(cfa, mainSignature).setReduceHeap(true).build()
-        bamCpaRun.execute()
-        val cache = bamCpaRun.cpa.getCache()
-        val last = cache
-            .get(mainSignature)
-            .map { it.reachedSet.asCollection().maxBy { (it as JvmValueAbstractState).programLocation.offset } }
-            .single() as JvmValueAbstractState
+        val cfa: JvmCfa = CfaUtil.createInterproceduralCfa(programClassPool)
+        val mainSignature = MethodSignature("Test", "main", "([Ljava/lang/String;)V")
+
+        val cache = runCpa(cfa, mainSignature)
+        val last = getLastState(cache, mainSignature)
+
         "Then there should be two fields in the JvmValueAbstractState" {
             last.staticFields.size shouldBe 2
         }
@@ -80,19 +88,96 @@ class CpaValueTest : FreeSpec({
             initialize = true,
         )
         val cfa: JvmCfa = CfaUtil.createInterproceduralCfa(programClassPool, libraryClassPool)
-        val clazz = programClassPool.getClass("Test") as ProgramClass
-        val mainSignature = Signature.of(clazz, clazz.findMethod("test", null)) as MethodSignature
-        val bamCpaRun = JvmValueBamCpaRun.Builder(cfa, mainSignature).setReduceHeap(true).build()
-        bamCpaRun.execute()
-        val cache = bamCpaRun.cpa.cache
-        val last = cache
-            .get(mainSignature)
-            .map { it.reachedSet.asCollection().maxBy { (it as JvmValueAbstractState).programLocation.offset } }
-            .single() as JvmValueAbstractState
+
+        val mainSignature = MethodSignature("Test", "test", "()V")
+        val cache = runCpa(cfa, mainSignature)
+        val last = getLastState(cache, mainSignature)
+
         "Correct string value" {
             val value = last.frame.localVariables[1].value
             value.shouldBeInstanceOf<ParticularReferenceValue>()
-            value.referenceValue().value() shouldBe "1"
+            value.referenceValue().value.preciseValue shouldBe "1"
+        }
+    }
+
+    "Checkcast instruction handled correctly in value analysis" - {
+        val (programClassPool, libraryClassPool) = ClassPoolBuilder.fromSource(
+            AssemblerSource(
+                "Test.jbc",
+                """
+                import java.lang.String;
+                import java.lang.StringBuilder;
+                version 8;
+                public class Test extends java.lang.Object {
+
+                    public void <init>() {
+                        aload_0
+                        invokespecial java.lang.Object#void <init>()
+                        return
+                    }
+
+                    public static String castOk() {
+                        new StringBuilder
+                        dup
+                        ldc "foo"
+                        invokespecial StringBuilder#void <init>(String)
+                        checkcast StringBuilder
+                        invokevirtual StringBuilder#String toString()
+                        areturn
+                    }
+
+                    public static String castInterface() {
+                        new StringBuilder
+                        dup
+                        ldc "foo"
+                        invokespecial StringBuilder#void <init>(String)
+                        checkcast java.lang.CharSequence
+                        invokevirtual StringBuilder#String toString()
+                        areturn
+                    }
+
+                    public static String castNotOk() {
+                        new StringBuilder
+                        dup
+                        ldc "foo"
+                        invokespecial StringBuilder#void <init>(String)
+                        checkcast java.lang.ClassLoader
+                        invokevirtual StringBuilder#String toString()
+                        areturn
+                    }
+                }
+                """.trimIndent(),
+            ),
+        )
+        val cfa: JvmCfa = CfaUtil.createInterproceduralCfa(programClassPool, libraryClassPool)
+
+        "Simple successful checkcast (same class)" - {
+            val mainSignature = MethodSignature("Test", "castOk", "()Ljava/lang/String;")
+            val cache = runCpa(cfa, mainSignature)
+            val last = getLastState(cache, mainSignature)
+
+            val value = last.frame.operandStack[0].value
+            value.shouldBeInstanceOf<ParticularReferenceValue>()
+            value.referenceValue().value.preciseValue shouldBe "foo"
+        }
+
+        "Successful checkcast (interface)" - {
+            val mainSignature = MethodSignature("Test", "castInterface", "()Ljava/lang/String;")
+            val cache = runCpa(cfa, mainSignature)
+            val last = getLastState(cache, mainSignature)
+
+            val value = last.frame.operandStack[0].value
+            value.shouldBeInstanceOf<ParticularReferenceValue>()
+            value.referenceValue().value.preciseValue shouldBe "foo"
+        }
+
+        "Unsuccessful cast" {
+            val mainSignature = MethodSignature("Test", "castNotOk", "()Ljava/lang/String;")
+            val cache = runCpa(cfa, mainSignature)
+            val last = getLastState(cache, mainSignature)
+
+            val value = last.frame.operandStack[0].value
+            value.shouldNotBeInstanceOf<ParticularReferenceValue>()
         }
     }
 })