diff --git a/src/mono/mono/mini/mini-llvm.c b/src/mono/mono/mini/mini-llvm.c index 4a4308dd555c3..859f135464b51 100644 --- a/src/mono/mono/mini/mini-llvm.c +++ b/src/mono/mono/mini/mini-llvm.c @@ -9499,47 +9499,32 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) } case OP_XOP_I4_I4: case OP_XOP_I8_I8: { - IntrinsicId id = (IntrinsicId)0; - switch (ins->inst_c0) { - case SIMD_OP_ARM64_RBIT32: id = INTRINS_BITREVERSE_I32; break; - case SIMD_OP_ARM64_RBIT64: id = INTRINS_BITREVERSE_I64; break; - default: g_assert_not_reached (); break; - } + IntrinsicId id = (IntrinsicId)ins->inst_c0; values [ins->dreg] = call_intrins (ctx, id, &lhs, ""); break; } case OP_XOP_X_X_X: case OP_XOP_I4_I4_I4: case OP_XOP_I4_I4_I8: { - IntrinsicId id = (IntrinsicId)0; + IntrinsicId id = (IntrinsicId)ins->inst_c0; gboolean zext_last = FALSE, bitcast_result = FALSE, getElement = FALSE; int element_idx = -1; - switch (ins->inst_c0) { - case SIMD_OP_ARM64_CRC32B: id = INTRINS_AARCH64_CRC32B; zext_last = TRUE; break; - case SIMD_OP_ARM64_CRC32H: id = INTRINS_AARCH64_CRC32H; zext_last = TRUE; break; - case SIMD_OP_ARM64_CRC32W: id = INTRINS_AARCH64_CRC32W; zext_last = TRUE; break; - case SIMD_OP_ARM64_CRC32X: id = INTRINS_AARCH64_CRC32X; break; - case SIMD_OP_ARM64_CRC32CB: id = INTRINS_AARCH64_CRC32CB; zext_last = TRUE; break; - case SIMD_OP_ARM64_CRC32CH: id = INTRINS_AARCH64_CRC32CH; zext_last = TRUE; break; - case SIMD_OP_ARM64_CRC32CW: id = INTRINS_AARCH64_CRC32CW; zext_last = TRUE; break; - case SIMD_OP_ARM64_CRC32CX: id = INTRINS_AARCH64_CRC32CX; break; - case SIMD_OP_AES_DEC: id = INTRINS_AARCH64_AESD; break; - case SIMD_OP_AES_ENC: id = INTRINS_AARCH64_AESE; break; - case SIMD_OP_ARM64_SHA1SU1: id = INTRINS_AARCH64_SHA1SU1; break; - case SIMD_OP_ARM64_SHA256SU0: id = INTRINS_AARCH64_SHA256SU0; break; - case SIMD_OP_ARM64_PMULL64_LOWER: - id = INTRINS_AARCH64_PMULL64; + switch (id) { + case INTRINS_AARCH64_PMULL64: getElement = TRUE; - element_idx = 0; bitcast_result = TRUE; + element_idx = ins->inst_c1; break; - case SIMD_OP_ARM64_PMULL64_UPPER: - id = INTRINS_AARCH64_PMULL64; - getElement = TRUE; - element_idx = 1; - bitcast_result = TRUE; + case INTRINS_AARCH64_CRC32B: + case INTRINS_AARCH64_CRC32H: + case INTRINS_AARCH64_CRC32W: + case INTRINS_AARCH64_CRC32CB: + case INTRINS_AARCH64_CRC32CH: + case INTRINS_AARCH64_CRC32CW: + zext_last = TRUE; + break; + default: break; - default: g_assert_not_reached (); break; } LLVMValueRef arg1 = rhs; if (zext_last) @@ -9555,18 +9540,18 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_XOP_X_X_X_X: { - IntrinsicId id = (IntrinsicId)0; + IntrinsicId id = (IntrinsicId)ins->inst_c0; gboolean getLowerElement = FALSE; int arg_idx = -1; - switch (ins->inst_c0) { - case SIMD_OP_ARM64_SHA1SU0: id = INTRINS_AARCH64_SHA1SU0; break; - case SIMD_OP_ARM64_SHA256H: id = INTRINS_AARCH64_SHA256H; break; - case SIMD_OP_ARM64_SHA256H2: id = INTRINS_AARCH64_SHA256H2; break; - case SIMD_OP_ARM64_SHA256SU1: id = INTRINS_AARCH64_SHA256SU1; break; - case SIMD_OP_ARM64_SHA1C: id = INTRINS_AARCH64_SHA1C; getLowerElement = TRUE; arg_idx = 1; break; - case SIMD_OP_ARM64_SHA1M: id = INTRINS_AARCH64_SHA1M; getLowerElement = TRUE; arg_idx = 1; break; - case SIMD_OP_ARM64_SHA1P: id = INTRINS_AARCH64_SHA1P; getLowerElement = TRUE; arg_idx = 1; break; - default: g_assert_not_reached (); break; + switch (id) { + case INTRINS_AARCH64_SHA1C: + case INTRINS_AARCH64_SHA1M: + case INTRINS_AARCH64_SHA1P: + getLowerElement = TRUE; + arg_idx = 1; + break; + default: + break; } LLVMValueRef args [] = { lhs, rhs, arg3 }; if (getLowerElement) @@ -9575,17 +9560,12 @@ process_bb (EmitContext *ctx, MonoBasicBlock *bb) break; } case OP_XOP_X_X: { - IntrinsicId id = (IntrinsicId)0; + IntrinsicId id = (IntrinsicId)ins->inst_c0; LLVMTypeRef ret_t = simd_class_to_llvm_type (ctx, ins->klass); gboolean getLowerElement = FALSE; - switch (ins->opcode) { - default: - switch (ins->inst_c0) { - case SIMD_OP_AES_IMC: id = INTRINS_AARCH64_AESIMC; break; - case SIMD_OP_ARM64_AES_AESMC: id = INTRINS_AARCH64_AESMC; break; - case SIMD_OP_ARM64_SHA1H: id = INTRINS_AARCH64_SHA1H; getLowerElement = TRUE; break; - default: g_assert_not_reached (); break; - } + switch (id) { + case INTRINS_AARCH64_SHA1H: getLowerElement = TRUE; break; + default: break; } LLVMValueRef arg0 = lhs; if (getLowerElement) diff --git a/src/mono/mono/mini/mini-ops.h b/src/mono/mono/mini/mini-ops.h index d07e902a0a20a..d7ab7c739f3c1 100644 --- a/src/mono/mono/mini/mini-ops.h +++ b/src/mono/mono/mini/mini-ops.h @@ -1540,7 +1540,7 @@ MINI_OP(OP_XBINOP, "xbinop", XREG, XREG, XREG) MINI_OP(OP_XBINOP_FORCEINT, "xbinop_forceint", XREG, XREG, XREG) MINI_OP(OP_XBINOP_SCALAR, "xbinop_scalar", XREG, XREG, XREG) MINI_OP(OP_XBINOP_BYSCALAR, "xbinop_byscalar", XREG, XREG, XREG) -/* inst_c0 contains a SimdOp, inst_c1 might contain additional data */ +/* inst_c0 contains an INTRINS_ enum, inst_c1 might contain additional data */ MINI_OP(OP_XOP, "xop", NONE, NONE, NONE) MINI_OP(OP_XOP_X_I, "xop_x_i", XREG, IREG, NONE) MINI_OP(OP_XOP_X_X, "xop_x_x", XREG, XREG, NONE) diff --git a/src/mono/mono/mini/mini.h b/src/mono/mono/mini/mini.h index eea7742708de4..d95c4b2cee844 100644 --- a/src/mono/mono/mini/mini.h +++ b/src/mono/mono/mini/mini.h @@ -2893,38 +2893,6 @@ enum { SIMD_PREFETCH_MODE_2, }; -/* SIMD operations */ -typedef enum { - SIMD_OP_AES_IMC, - SIMD_OP_AES_ENC, - SIMD_OP_AES_ENCLAST, - SIMD_OP_AES_DEC, - SIMD_OP_AES_DECLAST, - SIMD_OP_ARM64_CRC32B, - SIMD_OP_ARM64_CRC32H, - SIMD_OP_ARM64_CRC32W, - SIMD_OP_ARM64_CRC32X, - SIMD_OP_ARM64_CRC32CB, - SIMD_OP_ARM64_CRC32CH, - SIMD_OP_ARM64_CRC32CW, - SIMD_OP_ARM64_CRC32CX, - SIMD_OP_ARM64_RBIT32, - SIMD_OP_ARM64_RBIT64, - SIMD_OP_ARM64_AES_AESMC, - SIMD_OP_ARM64_SHA1C, - SIMD_OP_ARM64_SHA1H, - SIMD_OP_ARM64_SHA1M, - SIMD_OP_ARM64_SHA1P, - SIMD_OP_ARM64_SHA1SU0, - SIMD_OP_ARM64_SHA1SU1, - SIMD_OP_ARM64_SHA256H, - SIMD_OP_ARM64_SHA256H2, - SIMD_OP_ARM64_SHA256SU0, - SIMD_OP_ARM64_SHA256SU1, - SIMD_OP_ARM64_PMULL64_LOWER, - SIMD_OP_ARM64_PMULL64_UPPER, -} SimdOp; - const char *mono_arch_xregname (int reg); MonoCPUFeatures mono_arch_get_cpu_features (void); diff --git a/src/mono/mono/mini/simd-intrinsics.c b/src/mono/mono/mini/simd-intrinsics.c index 40df6910a9181..d511ee4663e8c 100644 --- a/src/mono/mono/mini/simd-intrinsics.c +++ b/src/mono/mono/mini/simd-intrinsics.c @@ -1054,30 +1054,30 @@ static SimdIntrinsic crc32_methods [] = { }; static SimdIntrinsic crypto_aes_methods [] = { - {SN_Decrypt, OP_XOP_X_X_X, SIMD_OP_AES_DEC}, - {SN_Encrypt, OP_XOP_X_X_X, SIMD_OP_AES_ENC}, - {SN_InverseMixColumns, OP_XOP_X_X, SIMD_OP_AES_IMC}, - {SN_MixColumns, OP_XOP_X_X, SIMD_OP_ARM64_AES_AESMC}, - {SN_PolynomialMultiplyWideningLower, OP_XOP_X_X_X, SIMD_OP_ARM64_PMULL64_LOWER}, - {SN_PolynomialMultiplyWideningUpper, OP_XOP_X_X_X, SIMD_OP_ARM64_PMULL64_UPPER}, + {SN_Decrypt, OP_XOP_X_X_X, INTRINS_AARCH64_AESD}, + {SN_Encrypt, OP_XOP_X_X_X, INTRINS_AARCH64_AESE}, + {SN_InverseMixColumns, OP_XOP_X_X, INTRINS_AARCH64_AESIMC}, + {SN_MixColumns, OP_XOP_X_X, INTRINS_AARCH64_AESMC}, + {SN_PolynomialMultiplyWideningLower}, + {SN_PolynomialMultiplyWideningUpper}, {SN_get_IsSupported}, }; static SimdIntrinsic sha1_methods [] = { - {SN_FixedRotate, OP_XOP_X_X, SIMD_OP_ARM64_SHA1H}, - {SN_HashUpdateChoose, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA1C}, - {SN_HashUpdateMajority, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA1M}, - {SN_HashUpdateParity, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA1P}, - {SN_ScheduleUpdate0, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA1SU0}, - {SN_ScheduleUpdate1, OP_XOP_X_X_X, SIMD_OP_ARM64_SHA1SU1}, + {SN_FixedRotate, OP_XOP_X_X, INTRINS_AARCH64_SHA1H}, + {SN_HashUpdateChoose, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA1C}, + {SN_HashUpdateMajority, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA1M}, + {SN_HashUpdateParity, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA1P}, + {SN_ScheduleUpdate0, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA1SU0}, + {SN_ScheduleUpdate1, OP_XOP_X_X_X, INTRINS_AARCH64_SHA1SU1}, {SN_get_IsSupported} }; static SimdIntrinsic sha256_methods [] = { - {SN_HashUpdate1, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA256H}, - {SN_HashUpdate2, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA256H2}, - {SN_ScheduleUpdate0, OP_XOP_X_X_X, SIMD_OP_ARM64_SHA256SU0}, - {SN_ScheduleUpdate1, OP_XOP_X_X_X_X, SIMD_OP_ARM64_SHA256SU1}, + {SN_HashUpdate1, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA256H}, + {SN_HashUpdate2, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA256H2}, + {SN_ScheduleUpdate0, OP_XOP_X_X_X, INTRINS_AARCH64_SHA256SU0}, + {SN_ScheduleUpdate1, OP_XOP_X_X_X_X, INTRINS_AARCH64_SHA256SU1}, {SN_get_IsSupported} }; @@ -1488,7 +1488,7 @@ emit_arm64_intrinsics ( case SN_ReverseElementBits: return emit_simd_ins_for_sig (cfg, klass, (is_64bit ? OP_XOP_I8_I8 : OP_XOP_I4_I4), - (is_64bit ? SIMD_OP_ARM64_RBIT64 : SIMD_OP_ARM64_RBIT32), + (is_64bit ? INTRINS_BITREVERSE_I64 : INTRINS_BITREVERSE_I32), arg0_type, fsig, args); default: g_assert_not_reached (); // if a new API is added we need to either implement it or change IsSupported to false @@ -1499,13 +1499,13 @@ emit_arm64_intrinsics ( switch (id) { case SN_ComputeCrc32: case SN_ComputeCrc32C: { - SimdOp op = (SimdOp)0; + IntrinsicId op = (IntrinsicId)0; gboolean is_c = info->id == SN_ComputeCrc32C; switch (get_underlying_type (fsig->params [1])) { - case MONO_TYPE_U1: op = is_c ? SIMD_OP_ARM64_CRC32CB : SIMD_OP_ARM64_CRC32B; break; - case MONO_TYPE_U2: op = is_c ? SIMD_OP_ARM64_CRC32CH : SIMD_OP_ARM64_CRC32H; break; - case MONO_TYPE_U4: op = is_c ? SIMD_OP_ARM64_CRC32CW : SIMD_OP_ARM64_CRC32W; break; - case MONO_TYPE_U8: op = is_c ? SIMD_OP_ARM64_CRC32CX : SIMD_OP_ARM64_CRC32X; break; + case MONO_TYPE_U1: op = is_c ? INTRINS_AARCH64_CRC32CB : INTRINS_AARCH64_CRC32B; break; + case MONO_TYPE_U2: op = is_c ? INTRINS_AARCH64_CRC32CH : INTRINS_AARCH64_CRC32H; break; + case MONO_TYPE_U4: op = is_c ? INTRINS_AARCH64_CRC32CW : INTRINS_AARCH64_CRC32W; break; + case MONO_TYPE_U8: op = is_c ? INTRINS_AARCH64_CRC32CX : INTRINS_AARCH64_CRC32X; break; default: g_assert_not_reached (); break; } return emit_simd_ins_for_sig (cfg, klass, is_64bit ? OP_XOP_I4_I4_I8 : OP_XOP_I4_I4_I4, op, arg0_type, fsig, args); @@ -1751,6 +1751,17 @@ emit_arm64_intrinsics ( } } + if (feature == MONO_CPU_ARM64_CRYPTO) { + switch (id) { + case SN_PolynomialMultiplyWideningLower: + return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, INTRINS_AARCH64_PMULL64, 0, fsig, args); + case SN_PolynomialMultiplyWideningUpper: + return emit_simd_ins_for_sig (cfg, klass, OP_XOP_X_X_X, INTRINS_AARCH64_PMULL64, 1, fsig, args); + default: + g_assert_not_reached (); + } + } + return NULL; } #endif // TARGET_ARM64