From 9203d7c02bce52289b911177284ff1e383348940 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:22:56 -0400 Subject: [PATCH 1/9] Move helpers --- src/coreclr/jit/emitarm64.cpp | 8475 +++++++----------------------- src/coreclr/jit/emitarm64.h | 10 +- src/coreclr/jit/emitarm64sve.cpp | 4706 ++++++++++++++++- 3 files changed, 6598 insertions(+), 6593 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 0777507aec45f..a17a5990b7495 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -1900,7 +1900,7 @@ void emitter::emitInsSanityCheck(instrDesc* id) { ssize_t imm1; ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); assert(insOptsScalableStandard(id->idInsOpt())); assert(isVectorRegister(id->idReg1())); // ddddd assert(isValidSimm<5>(imm1)); // iiiii @@ -3381,18 +3381,6 @@ static const char * const pnRegNames[] = "pn15" }; -static const char * const svePatternNames[] = -{ - "pow2", "vl1", "vl2", "vl3", - "vl4", "vl5", "vl6", "vl7", - "vl8", "vl16", "vl32", "vl64", - "vl128", "vl256", "invalid", "invalid", - "invalid", "invalid", "invalid", "invalid", - "invalid", "invalid", "invalid", "invalid", - "invalid", "invalid", "invalid", "invalid", - "invalid", "mul4", "mul3", "all" -}; - // clang-format on //------------------------------------------------------------------------ @@ -13693,33 +13681,6 @@ void emitter::emitIns_Call(EmitCallType callType, } } -/***************************************************************************** - * - * Returns the encoding to set the vector length specifier (vl) for an Arm64 SVE instruction - */ - -/*static*/ emitter::code_t emitter::insEncodeVectorLengthSpecifier(instrDesc* id) -{ - assert(id != nullptr); - assert(insOptsScalableStandard(id->idInsOpt())); - - if (id->idVectorLength4x()) - { - switch (id->idInsFmt()) - { - case IF_SVE_DL_2A: - return 0x400; // set the bit at location 10 - case IF_SVE_DY_3A: - return 0x2000; // set the bit at location 13 - default: - assert(!"Unexpected format"); - break; - } - } - - return 0; -} - /***************************************************************************** * * Returns the encoding to select 'index' for an Arm64 vector elem instruction @@ -13782,18 +13743,6 @@ void emitter::emitIns_Call(EmitCallType callType, return (bits << 11); // bits at locations [14,13,12,11] } -/***************************************************************************** - * - * Returns the encoding for an immediate in the SVE variant of dup (indexed) - */ -/*static*/ emitter::code_t emitter::insEncodeSveBroadcastIndex(emitAttr elemsize, ssize_t index) -{ - unsigned lane_bytes = genLog2(elemsize) + 1; - code_t tsz = (1 << (lane_bytes - 1)); - code_t imm = (code_t)index << lane_bytes | tsz; - return insEncodeSplitUimm<23, 22, 20, 16>(imm); -} - /***************************************************************************** * * Returns the encoding to select the 'index' for an Arm64 'mul' by element instruction @@ -14280,6552 +14229,2341 @@ void emitter::emitIns_Call(EmitCallType callType, /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * Returns the encoding for the immediate value as 9-bits at bit locations '21-16' for high and '12-10' for low. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSimm9h9l_21_to_16_and_12_to_10(ssize_t imm) { - switch (size) - { - case EA_1BYTE: - return 0x00000000; - - case EA_2BYTE: - return 0x00400000; // set the bit at location 22 + assert(isValidSimm<9>(imm)); - case EA_4BYTE: - return 0x00800000; // set the bit at location 23 + if (imm < 0) + { + imm = (imm & 0x1FF); + } - case EA_8BYTE: - return 0x00C00000; // set the bit at location 23 and 22 + code_t h = (code_t)(imm & 0x1F8) << 13; // encode high 6-bits at locations '21-16' + code_t l = (code_t)((imm & ~0x1F8) & 0x7) << 10; // encode low 3-bits at locations '12-10' - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + return (h | l); } /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the size at bit locations '22-21'. + * Returns the encoding for the immediate value that is a multiple of 2 as 4-bits at bit locations '19-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_22_to_21(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf2_19_to_16(ssize_t imm) { - switch (size) - { - case EA_1BYTE: - return 0; - - case EA_2BYTE: - return (1 << 21); // set the bit at location 21 - - case EA_4BYTE: - return (1 << 22); // set the bit at location 22 - - case EA_8BYTE: - return (1 << 22) | (1 << 21); // set the bit at location 22 and 21 - - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + assert((isValidSimm_MultipleOf<4, 2>(imm))); + return insEncodeSimm<19, 16>(imm / 2); } /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the size at bit locations '18-17'. + * Returns the encoding for the immediate value that is a multiple of 3 as 4-bits at bit locations '19-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_18_to_17(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf3_19_to_16(ssize_t imm) { - switch (size) - { - case EA_1BYTE: - return 0; - - case EA_2BYTE: - return (1 << 17); // set the bit at location 17 - - case EA_4BYTE: - return (1 << 18); // set the bit at location 18 - - case EA_8BYTE: - return (1 << 18) | (1 << 17); // set the bit at location 18 and 17 - - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + assert((isValidSimm_MultipleOf<4, 3>(imm))); + return insEncodeSimm<19, 16>(imm / 3); } /***************************************************************************** * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'sz' at bit location '20'. + * Returns the encoding for the immediate value that is a multiple of 4 as 4-bits at bit locations '19-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_20(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf4_19_to_16(ssize_t imm) { - switch (size) - { - case EA_4BYTE: - return 0; - - case EA_8BYTE: - return (1 << 20); - - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + assert((isValidSimm_MultipleOf<4, 4>(imm))); + return insEncodeSimm<19, 16>(imm / 4); } /***************************************************************************** * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'sz' at bit location '21'. + * Returns the encoding for the immediate value that is a multiple of 16 as 4-bits at bit locations '19-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_21(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf16_19_to_16(ssize_t imm) { - switch (size) - { - case EA_4BYTE: - return 0; - - case EA_8BYTE: - return (1 << 21); - - default: - assert(!"Invalid insOpt for vector register"); - } - return 0; + assert((isValidSimm_MultipleOf<4, 16>(imm))); + return insEncodeSimm<19, 16>(imm / 16); } /***************************************************************************** * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:20-19'. + * Returns the encoding for the immediate value that is a multiple of 32 as 4-bits at bit locations '19-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_23_tszl_20_to_19(emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf32_19_to_16(ssize_t imm) { - switch (size) - { - case EA_1BYTE: - return 0x080000; // set the bit at location 19 - - case EA_2BYTE: - return 0x100000; // set the bit at location 20 - - case EA_4BYTE: - return 0x400000; // set the bit at location 22 - - case EA_8BYTE: - return 0x800000; // set the bit at location 23 - - default: - assert(!"Invalid size for vector register"); - } - return 0; + assert((isValidSimm_MultipleOf<4, 32>(imm))); + return insEncodeSimm<19, 16>(imm / 32); } /***************************************************************************** * - * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction at bit location '30'. - * This only works on select formats. + * Returns the encoding for the immediate value that is a multiple of 2 as 5-bits at bit locations '20-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_30_or_21(insFormat fmt, emitAttr size) +/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf2_20_to_16(ssize_t imm) { - switch (fmt) - { - case IF_SVE_HX_3A_B: - case IF_SVE_HX_3A_E: - switch (size) - { - case EA_4BYTE: - return 0; - - case EA_8BYTE: - return (1 << 30); - - default: - break; - } - - assert(!"Invalid size for vector register"); - return 0; - - case IF_SVE_IV_3A: - assert(size == EA_8BYTE); - return 0; - - case IF_SVE_JI_3A_A: - switch (size) - { - case EA_4BYTE: - return (1 << 21); - - case EA_8BYTE: - return 0; - - default: - break; - } - - assert(!"Invalid size for vector register"); - return 0; - - default: - break; - } - - assert(!"Unexpected instruction format"); - return 0; + assert((isValidUimm_MultipleOf<5, 2>(imm))); + return insEncodeUimm<20, 16>(imm / 2); } + /***************************************************************************** * - * Returns the encoding for the field 'i1:tszh:tszl' at bit locations '23-22:20-18'. + * Returns the encoding for the immediate value that is a multiple of 4 as 5-bits at bit locations '20-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_tszl_and_imm(const insOpts opt, const ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm) { - code_t encoding = 0; + assert((isValidUimm_MultipleOf<5, 4>(imm))); + return insEncodeUimm<20, 16>(imm / 4); +} - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - encoding = 0x040000; // set the bit at location 18 - // encode immediate at location 23-22:20-19 - encoding |= ((imm & 0b1100) << 22); - encoding |= ((imm & 0b11) << 19); - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. + */ - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - encoding = 0x080000; // set the bit at location 19 - // encode immediate at location 23-22:20 - encoding |= ((imm & 0b110) << 22); - encoding |= ((imm & 1) << 20); - break; +/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<5, 8>(imm))); + return insEncodeUimm<20, 16>(imm / 8); +} - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - encoding = 0x100000; // set the bit at location 20 - encoding |= (imm << 22); // encode immediate at location 23:22 - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. + */ - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - encoding = 0x400000; // set the bit at location 22 - encoding |= (imm << 23); // encode immediate at location 23 - break; +/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf2_21_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<6, 2>(imm))); + return insEncodeUimm<21, 16>(imm / 2); +} - default: - assert(!"Invalid size for vector register"); - break; - } +/***************************************************************************** + * + * Returns the encoding for the immediate value that is a multiple of 4 as 6-bits at bit locations '21-16'. + */ - return encoding; +/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf4_21_to_16(ssize_t imm) +{ + assert((isValidUimm_MultipleOf<6, 4>(imm))); + return insEncodeUimm<21, 16>(imm / 4); } /***************************************************************************** * - * Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. + * Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, - ssize_t imm, - bool isRightShift) +/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf8_21_to_16(ssize_t imm) { - code_t encoding = 0; - - imm = insEncodeShiftImmediate(optGetSveElemsize(opt), isRightShift, imm); + assert((isValidUimm_MultipleOf<6, 8>(imm))); + return insEncodeUimm<21, 16>(imm / 8); +} - switch (opt) - { - case INS_OPTS_SCALABLE_B: - imm = imm & 0b111; // bits 18-16 - encoding |= (1 << 19); // bit 19 - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 1-bit at bit locations '23'. + */ - case INS_OPTS_SCALABLE_H: - imm = imm & 0b1111; // bits 19-16 - encoding |= (1 << 20); // bit 20 - break; +/*static*/ emitter::code_t emitter::insEncodeUimm1_23(ssize_t imm) +{ + assert(isValidUimm<1>(imm)); + return (code_t)imm << 23; +} - case INS_OPTS_SCALABLE_S: - imm = imm & 0b11111; // bits 20-16 - encoding |= (1 << 22); // bit 22 - break; +/***************************************************************************** + * + * Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. + */ - case INS_OPTS_SCALABLE_D: - // this gets the last bit of 'imm' and tries to set bit 22 - encoding |= ((imm >> 5) << 22); - imm = imm & 0b11111; // bits 20-16 - encoding |= (1 << 23); // bit 23 - break; +/*static*/ emitter::code_t emitter::insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm) +{ + assert(isValidUimm<3>(imm)); - default: - assert(!"Invalid size for vector register"); - break; - } + code_t h = (code_t)(imm & 0x6) << 21; // encode high 2-bits at locations '23-22' + code_t l = (code_t)(imm & 0x1) << 12; // encode low 1-bit at locations '12' - return (encoding | (code_t)(imm << 16)); + return (h | l); } /***************************************************************************** * - * Returns the encoding for the field 'i1:tsz' at bit locations '20:19-16'. + * Returns the encoding for the immediate value as 4-bits starting from 1, at bit locations '19-16'. */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithImmediate_i1_tsz(const insOpts opt, ssize_t imm) +/*static*/ emitter::code_t emitter::insEncodeUimm4From1_19_to_16(ssize_t imm) { - code_t encoding = 0; - - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimm<4>(imm)); - encoding |= (1 << 16); // bit 16 - encoding |= (imm << 17); // bits 20-17 - break; - - case INS_OPTS_SCALABLE_H: - assert(isValidUimm<3>(imm)); - encoding |= (1 << 17); // bit 17 - encoding |= (imm << 18); // bits 20-18 - break; - - case INS_OPTS_SCALABLE_S: - assert(isValidUimm<2>(imm)); - encoding |= (1 << 18); // bit 18 - encoding |= (imm << 19); // bits 20-19 - break; - - case INS_OPTS_SCALABLE_D: - assert(isValidUimm<1>(imm)); - encoding |= (1 << 19); // bit 19 - encoding |= (imm << 20); // bit 20 - break; - - default: - assert(!"Invalid size for vector register"); - break; - } - - return encoding; + assert(isValidUimmFrom1<4>(imm)); + return (code_t)(imm - 1) << 16; } /***************************************************************************** * - * Returns the encoding to select the elemsize for an Arm64 SVE vector instruction plus an immediate. - * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:9-8'. + * Returns the encoding for the immediate value as 8-bits at bit locations '12-5'. */ -/*static*/ emitter::code_t emitter::insEncodeSveShift_23_to_22_9_to_0(emitAttr size, bool isRightShift, size_t imm) +/*static*/ emitter::code_t emitter::insEncodeImm8_12_to_5(ssize_t imm) { - code_t encodedSize = 0; + assert(isValidSimm<8>(imm) || isValidUimm<8>(imm)); + return (code_t)((imm & 0xFF) << 5); +} - switch (size) +BYTE* emitter::emitOutputLoadLabel(BYTE* dst, BYTE* srcAddr, BYTE* dstAddr, instrDescJmp* id) +{ + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); + regNumber dstReg = id->idReg1(); + if (id->idjShort) { - case EA_1BYTE: - encodedSize = 0x100; // set the bit at location 8 - break; - - case EA_2BYTE: - encodedSize = 0x200; // set the bit at location 9 - break; - - case EA_4BYTE: - encodedSize = 0x400000; // set the bit at location 22 - break; - - case EA_8BYTE: - encodedSize = 0x800000; // set the bit at location 23 - break; - - default: - assert(!"Invalid esize for vector register"); + // adr x, [rel addr] -- compute address: current addr(ip) + rel addr. + assert(ins == INS_adr); + assert(fmt == IF_DI_1E); + ssize_t distVal = (ssize_t)(dstAddr - srcAddr); + dst = emitOutputShortAddress(dst, ins, fmt, distVal, dstReg); } + else + { + // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr + assert(fmt == IF_LARGEADR); + ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); + dst = emitOutputShortAddress(dst, INS_adrp, IF_DI_1E, relPageAddr, dstReg); - code_t encodedImm = insEncodeShiftImmediate(size, isRightShift, imm); - code_t imm3High = (encodedImm & 0x60) << 17; - code_t imm3Low = (encodedImm & 0x1f) << 5; - return encodedSize | imm3High | imm3Low; -} - -/***************************************************************************** - * - * Returns the encoding to select the constant values 90 or 270 for an Arm64 SVE vector instruction - * This specifically encode the field 'rot' at bit location '16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveImm90_or_270_rot(ssize_t imm) -{ - assert(emitIsValidEncodedRotationImm90_or_270(imm)); - return (code_t)(imm << 16); + // add x, x, page offs -- compute address = page addr + page offs + ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits + assert(isValidUimm<12>(imm12)); + code_t code = + emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) + code |= insEncodeDatasize(EA_8BYTE); // X + code |= ((code_t)imm12 << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(dstReg); // ddddd + code |= insEncodeReg_Rn(dstReg); // nnnnn + dst += emitOutput_Instr(dst, code); + } + return dst; } /***************************************************************************** * - * Returns the encoding to select the constant values 0, 90, 180 or 270 for an Arm64 SVE vector instruction - * This specifically encode the field 'rot' at bit locations '14-13'. + * Output a local jump or other instruction with a pc-relative immediate. + * Note that this may be invoked to overwrite an existing jump instruction at 'dst' + * to handle forward branch patching. */ -/*static*/ emitter::code_t emitter::insEncodeSveImm0_to_270_rot(ssize_t imm) +BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) { - assert(emitIsValidEncodedRotationImm0_to_270(imm)); - return (code_t)(imm << 13); -} + instrDescJmp* id = (instrDescJmp*)i; -/***************************************************************************** - * - * Returns the encoding to select the constant float values 0, 0.5, 1.0 or 2.0 for an Arm64 SVE vector instruction - * This specifically encode the field 'i1' at bit location '5'. - */ + unsigned srcOffs; + unsigned dstOffs; + BYTE* srcAddr; + BYTE* dstAddr; + ssize_t distVal; -/*static*/ emitter::code_t emitter::insEncodeSveSmallFloatImm(ssize_t imm) -{ - assert(emitIsValidEncodedSmallFloatImm(imm)); - return (code_t)(imm << 5); -} + // Set default ins/fmt from id. + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); -/***************************************************************************** - * - * Returns the register list size for the given SVE instruction. - */ + bool loadLabel = false; + bool isJump = false; + bool loadConstant = false; -/*static*/ int emitter::insGetSveReg1ListSize(instruction ins) -{ switch (ins) { - case INS_sve_ld1d: - case INS_sve_ld1w: - case INS_sve_ld1sw: - case INS_sve_ld1sb: - case INS_sve_ld1b: - case INS_sve_ld1sh: - case INS_sve_ld1h: - case INS_sve_ldnf1d: - case INS_sve_ldnf1sw: - case INS_sve_ldnf1sh: - case INS_sve_ldnf1w: - case INS_sve_ldnf1h: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1b: - case INS_sve_ldnt1b: - case INS_sve_ldnt1d: - case INS_sve_ldnt1h: - case INS_sve_ldnt1w: - case INS_sve_ld1rob: - case INS_sve_ld1rod: - case INS_sve_ld1roh: - case INS_sve_ld1row: - case INS_sve_ld1rqb: - case INS_sve_ld1rqd: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - case INS_sve_stnt1b: - case INS_sve_stnt1d: - case INS_sve_stnt1h: - case INS_sve_stnt1w: - case INS_sve_st1d: - case INS_sve_st1w: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - case INS_sve_ldff1h: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - case INS_sve_st1b: - case INS_sve_st1h: - case INS_sve_ldff1sb: - case INS_sve_ldff1b: - case INS_sve_ldnt1sb: - case INS_sve_ldnt1sh: - case INS_sve_ld1rd: - case INS_sve_ld1rsw: - case INS_sve_ld1rh: - case INS_sve_ld1rsb: - case INS_sve_ld1rsh: - case INS_sve_ld1rw: - case INS_sve_ld1q: - case INS_sve_ldnt1sw: - case INS_sve_st1q: - case INS_sve_ld1rb: - return 1; - - case INS_sve_ld2b: - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - case INS_sve_ld2q: - case INS_sve_splice: // SVE_CV_3A - case INS_sve_st2b: - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - case INS_sve_st2q: - case INS_sve_whilege: // SVE_DX_3A - case INS_sve_whilegt: // SVE_DX_3A - case INS_sve_whilehi: // SVE_DX_3A - case INS_sve_whilehs: // SVE_DX_3A - case INS_sve_whilele: // SVE_DX_3A - case INS_sve_whilels: // SVE_DX_3A - case INS_sve_whilelt: // SVE_DX_3A - case INS_sve_pext: // SVE_DW_2B - return 2; - - case INS_sve_ld3b: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - case INS_sve_ld3q: - case INS_sve_st3b: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - case INS_sve_st3q: - return 3; - - case INS_sve_ld4b: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - case INS_sve_ld4q: - case INS_sve_st4b: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - case INS_sve_st4q: - return 4; - default: - assert(!"Unexpected instruction"); - return 1; - } -} + isJump = true; + break; -/***************************************************************************** - * - * Returns the predicate type for the given SVE format. - */ + case INS_tbz: + case INS_tbnz: + case INS_cbz: + case INS_cbnz: + isJump = true; + break; -/*static*/ emitter::PredicateType emitter::insGetPredicateType(insFormat fmt, int regpos /* =0 */) -{ - switch (fmt) - { - case IF_SVE_BV_2A: - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - case IF_SVE_HW_4A_B: - case IF_SVE_HW_4A_C: - case IF_SVE_HW_4B: - case IF_SVE_HW_4B_D: - case IF_SVE_HX_3A_E: - case IF_SVE_IJ_3A_D: - case IF_SVE_IJ_3A_E: - case IF_SVE_IJ_3A_F: - case IF_SVE_IK_4A_G: - case IF_SVE_IJ_3A_G: - case IF_SVE_IK_4A_I: - case IF_SVE_IH_3A_F: - case IF_SVE_II_4A_H: - case IF_SVE_IH_3A: - case IF_SVE_IH_3A_A: - case IF_SVE_II_4A: - case IF_SVE_II_4A_B: - case IF_SVE_IU_4A: - case IF_SVE_IU_4A_C: - case IF_SVE_IU_4B: - case IF_SVE_IU_4B_D: - case IF_SVE_IV_3A: - case IF_SVE_IG_4A_F: - case IF_SVE_IG_4A_G: - case IF_SVE_IJ_3A: - case IF_SVE_IK_4A: - case IF_SVE_IK_4A_F: - case IF_SVE_IK_4A_H: - case IF_SVE_IU_4A_A: - case IF_SVE_IU_4B_B: - case IF_SVE_HX_3A_B: - case IF_SVE_IG_4A: - case IF_SVE_IG_4A_D: - case IF_SVE_IG_4A_E: - case IF_SVE_IF_4A: - case IF_SVE_IF_4A_A: - case IF_SVE_IM_3A: - case IF_SVE_IN_4A: - case IF_SVE_IX_4A: - case IF_SVE_IO_3A: - case IF_SVE_IP_4A: - case IF_SVE_IQ_3A: - case IF_SVE_IR_4A: - case IF_SVE_IS_3A: - case IF_SVE_IT_4A: - case IF_SVE_GI_4A: - case IF_SVE_IC_3A_C: - case IF_SVE_IC_3A: - case IF_SVE_IC_3A_B: - case IF_SVE_IC_3A_A: - case IF_SVE_IL_3A_C: - case IF_SVE_IL_3A: - case IF_SVE_IL_3A_B: - case IF_SVE_IL_3A_A: - case IF_SVE_IW_4A: - return PREDICATE_ZERO; - - case IF_SVE_BV_2A_J: - case IF_SVE_CP_3A: - case IF_SVE_CQ_3A: - case IF_SVE_AM_2A: - case IF_SVE_AN_3A: - case IF_SVE_AO_3A: - case IF_SVE_HL_3A: - case IF_SVE_HM_2A: - case IF_SVE_AA_3A: - case IF_SVE_BU_2A: - case IF_SVE_BV_2B: - case IF_SVE_HS_3A: - case IF_SVE_HP_3A: - case IF_SVE_HP_3B: - case IF_SVE_AR_4A: - case IF_SVE_BV_2A_A: - case IF_SVE_AB_3A: - case IF_SVE_ET_3A: - case IF_SVE_HU_4A: - case IF_SVE_HL_3B: - case IF_SVE_AD_3A: - case IF_SVE_AB_3B: - case IF_SVE_AE_3A: - case IF_SVE_EU_3A: - case IF_SVE_GT_4A: - case IF_SVE_AP_3A: - case IF_SVE_HO_3A: - case IF_SVE_HO_3B: - case IF_SVE_HO_3C: - case IF_SVE_GQ_3A: - case IF_SVE_HU_4B: - case IF_SVE_AQ_3A: - case IF_SVE_CU_3A: - case IF_SVE_AC_3A: - case IF_SVE_ER_3A: - case IF_SVE_GR_3A: - case IF_SVE_ES_3A: - case IF_SVE_HR_3A: - case IF_SVE_EP_3A: - case IF_SVE_GP_3A: - case IF_SVE_EQ_3A: - case IF_SVE_HQ_3A: - case IF_SVE_AS_4A: - case IF_SVE_CT_3A: - case IF_SVE_HV_4A: - return PREDICATE_MERGE; - - case IF_SVE_CZ_4A_A: - case IF_SVE_CZ_4A_L: - case IF_SVE_CE_2A: - case IF_SVE_CE_2B: - case IF_SVE_CE_2C: - case IF_SVE_CE_2D: - case IF_SVE_CF_2A: - case IF_SVE_CF_2B: - case IF_SVE_CF_2C: - case IF_SVE_CF_2D: - case IF_SVE_CI_3A: - case IF_SVE_CJ_2A: - case IF_SVE_DE_1A: - case IF_SVE_DH_1A: - case IF_SVE_DJ_1A: - case IF_SVE_DM_2A: - case IF_SVE_DN_2A: - case IF_SVE_DO_2A: - case IF_SVE_DP_2A: - case IF_SVE_DR_1A: - case IF_SVE_DT_3A: - case IF_SVE_DU_3A: - case IF_SVE_CK_2A: - return PREDICATE_SIZED; - - case IF_SVE_DB_3A: - // Second register could be ZERO or MERGE so handled at source. - assert(regpos != 2); - return PREDICATE_SIZED; - - case IF_SVE_DL_2A: - case IF_SVE_DY_3A: - case IF_SVE_DZ_1A: - return PREDICATE_N_SIZED; - - // This is a special case as the second register could be ZERO or MERGE. - // / - // Therefore, by default return NONE due to ambiguity. - case IF_SVE_AH_3A: - // TODO: Handle these cases. - assert(false); - break; - - case IF_SVE_JD_4B: - case IF_SVE_JD_4C: - case IF_SVE_JI_3A_A: - case IF_SVE_JJ_4A: - case IF_SVE_JJ_4A_B: - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - case IF_SVE_JJ_4B: - case IF_SVE_JJ_4B_E: - case IF_SVE_JN_3B: - case IF_SVE_JN_3C: - case IF_SVE_JD_4A: - case IF_SVE_JN_3A: - case IF_SVE_JD_4C_A: - case IF_SVE_JJ_4B_C: - case IF_SVE_JL_3A: - case IF_SVE_JN_3C_D: - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - case IF_SVE_HY_3B: - case IF_SVE_HZ_2A_B: - case IF_SVE_IA_2A: - case IF_SVE_IB_3A: - case IF_SVE_JK_4A: - case IF_SVE_JK_4A_B: - case IF_SVE_JK_4B: - case IF_SVE_IZ_4A: - case IF_SVE_IZ_4A_A: - case IF_SVE_JB_4A: - case IF_SVE_JM_3A: - case IF_SVE_CM_3A: - case IF_SVE_CN_3A: - case IF_SVE_CO_3A: - case IF_SVE_JA_4A: - case IF_SVE_CR_3A: - case IF_SVE_CS_3A: - case IF_SVE_CV_3A: - case IF_SVE_CV_3B: - case IF_SVE_DW_2A: // [] - case IF_SVE_DW_2B: // [] - case IF_SVE_JC_4A: - case IF_SVE_JO_3A: - case IF_SVE_JE_3A: - case IF_SVE_JF_4A: - case IF_SVE_AK_3A: - case IF_SVE_HE_3A: - case IF_SVE_AF_3A: - case IF_SVE_AG_3A: - case IF_SVE_AI_3A: - case IF_SVE_AJ_3A: - case IF_SVE_AL_3A: - case IF_SVE_CL_3A: - case IF_SVE_GS_3A: - case IF_SVE_HJ_3A: - case IF_SVE_IY_4A: - return PREDICATE_NONE; - - case IF_SVE_CX_4A: - case IF_SVE_CX_4A_A: - case IF_SVE_CY_3A: - case IF_SVE_CY_3B: - case IF_SVE_GE_4A: - case IF_SVE_HT_4A: - assert((regpos == 1) || (regpos == 2)); - return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_CZ_4A: - case IF_SVE_DA_4A: - case IF_SVE_DB_3B: - case IF_SVE_DC_3A: - assert((regpos >= 1) && (regpos <= 4)); - return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_CZ_4A_K: - assert((regpos >= 1) && (regpos <= 3)); - return (regpos == 2 ? PREDICATE_MERGE : PREDICATE_SIZED); - - case IF_SVE_DD_2A: - case IF_SVE_DF_2A: - assert((regpos >= 1) && (regpos <= 3)); - return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); - - case IF_SVE_DG_2A: - return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_DI_2A: - return (regpos == 1 ? PREDICATE_NONE : PREDICATE_SIZED); - - case IF_SVE_DK_3A: - assert((regpos == 2) || (regpos == 3)); - return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); - - case IF_SVE_HI_3A: - assert((regpos == 1) || (regpos == 2)); - return ((regpos == 2) ? PREDICATE_ZERO : PREDICATE_SIZED); - - case IF_SVE_DV_4A: - assert((regpos >= 1) && (regpos <= 3)); - return ((regpos == 3) ? PREDICATE_SIZED : PREDICATE_NONE); - - case IF_SVE_ID_2A: - case IF_SVE_JG_2A: - return PREDICATE_NONE; + case INS_ldr: + case INS_ldrsw: + loadConstant = true; + break; - default: + case INS_adr: + case INS_adrp: + loadLabel = true; break; } - assert(!"Unexpected instruction format"); - return PREDICATE_NONE; -} + /* Figure out the distance to the target */ -/***************************************************************************** - * - * Returns true if the SVE instruction has a LSL addr. - * This is for formats that have [, , LSL #N], [{, , LSL #N}] - */ -/*static*/ bool emitter::insSveIsLslN(instruction ins, insFormat fmt) -{ - switch (fmt) - { - case IF_SVE_JD_4A: - switch (ins) - { - case INS_sve_st1h: - return true; + srcOffs = emitCurCodeOffs(dst); + srcAddr = emitOffsetToPtr(srcOffs); - default: - break; - } - break; + if (id->idAddr()->iiaIsJitDataOffset()) + { + assert(loadConstant || loadLabel); + int doff = id->idAddr()->iiaGetJitDataOffset(); + assert(doff >= 0); + ssize_t imm = emitGetInsSC(id); + assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 - case IF_SVE_JD_4B: - switch (ins) - { - case INS_sve_st1w: - return true; + unsigned dataOffs = (unsigned)(doff + imm); + assert(dataOffs < emitDataSize()); + dstAddr = emitDataOffsetToPtr(dataOffs); - default: - break; - } - break; + regNumber dstReg = id->idReg1(); + regNumber addrReg = dstReg; // an integer register to compute long address. + emitAttr opSize = id->idOpSize(); - case IF_SVE_HW_4B: - switch (ins) + if (loadConstant) + { + if (id->idjShort) { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ld1w: - case INS_sve_ldff1w: - return true; - - default: - break; + // ldr x/v, [rel addr] -- load constant from current addr(ip) + rel addr. + assert(ins == INS_ldr); + assert(fmt == IF_LS_1A); + distVal = (ssize_t)(dstAddr - srcAddr); + dst = emitOutputShortConstant(dst, ins, fmt, distVal, dstReg, opSize); } - break; - - case IF_SVE_IG_4A: - switch (ins) + else { - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - return true; - - default: - break; - } - break; + // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr + assert(fmt == IF_LARGELDC); + ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); + if (isVectorRegister(dstReg)) + { + // Update addrReg with the reserved integer register + // since we cannot use dstReg (vector) to load constant directly from memory. - case IF_SVE_IG_4A_F: - switch (ins) - { - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; + // If loading a 16-byte value, we will need to load directly into dstReg. + // Thus, encode addrReg for the ld1 instruction. + if (opSize == EA_16BYTE) + { + addrReg = encodingSPtoZR(id->idReg2()); + } + else + { + addrReg = id->idReg2(); + } - default: - break; - } - break; + assert(isGeneralRegister(addrReg)); + } - case IF_SVE_IG_4A_G: - switch (ins) - { - case INS_sve_ldff1h: - return true; + ins = INS_adrp; + fmt = IF_DI_1E; + dst = emitOutputShortAddress(dst, ins, fmt, relPageAddr, addrReg); - default: - break; - } - break; + ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits + assert(isValidUimm<12>(imm12)); - case IF_SVE_II_4A: - case IF_SVE_II_4A_B: - switch (ins) - { - case INS_sve_ld1d: - return true; + // Special case: emit add + ld1 instructions for loading 16-byte data into vector register. + if (isVectorRegister(dstReg) && (opSize == EA_16BYTE)) + { + const emitAttr elemSize = EA_1BYTE; + const insOpts opt = optMakeArrangement(opSize, elemSize); - default: - break; - } - break; + assert(isGeneralRegisterOrSP(addrReg)); + assert(isValidVectorElemsize(elemSize)); + assert(isValidArrangement(opSize, opt)); - case IF_SVE_II_4A_H: - switch (ins) - { - case INS_sve_ld1w: - return true; - - default: - break; - } - break; + // Calculate page addr + page offs, then emit ld1 instruction. + dst = emitOutputVectorConstant(dst, imm12, dstReg, addrReg, opSize, elemSize); + } + else + { + // ldr x, [x, 0] -- load constant from address into integer register. + ins = INS_ldr; + fmt = IF_LS_2B; + dst = emitOutputShortConstant(dst, ins, fmt, imm12, addrReg, opSize); - case IF_SVE_IK_4A: - switch (ins) - { - case INS_sve_ld1sw: - return true; + // fmov v, d -- copy constant in integer register to vector register. + // This is needed only for vector constant. + if (addrReg != dstReg) + { + // fmov Vd,Rn DV_2I X00111100X100111 000000nnnnnddddd 1E27 0000 Vd,Rn + // (scalar, from general) + assert(isVectorRegister(dstReg) && isGeneralRegister(addrReg)); + ins = INS_fmov; + fmt = IF_DV_2I; + code_t code = emitInsCode(ins, fmt); - default: - break; + code |= insEncodeReg_Vd(dstReg); // ddddd + code |= insEncodeReg_Rn(addrReg); // nnnnn + if (id->idOpSize() == EA_8BYTE) + { + code |= 0x80400000; // X ... X + } + dst += emitOutput_Instr(dst, code); + } + } } - break; + } + else + { + assert(loadLabel); + dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); + } - case IF_SVE_IK_4A_G: - switch (ins) - { - case INS_sve_ld1sh: - return true; + return dst; + } - default: - break; - } - break; + assert(loadLabel || isJump); - case IF_SVE_IK_4A_I: - switch (ins) - { - case INS_sve_ld1h: - return true; + if (id->idAddr()->iiaHasInstrCount()) + { + assert(ig != NULL); + int instrCount = id->idAddr()->iiaGetInstrCount(); + unsigned insNum = emitFindInsNum(ig, id); + if (instrCount < 0) + { + // Backward branches using instruction count must be within the same instruction group. + assert(insNum + 1 >= (unsigned)(-instrCount)); + } - default: - break; - } - break; + dstOffs = ig->igOffs + emitFindOffset(ig, (insNum + 1 + instrCount)); + dstAddr = emitOffsetToPtr(dstOffs); + } + else + { + dstOffs = id->idAddr()->iiaIGlabel->igOffs; + dstAddr = emitOffsetToPtr(dstOffs); + } - case IF_SVE_IN_4A: - switch (ins) - { - case INS_sve_ldnt1d: - case INS_sve_ldnt1h: - case INS_sve_ldnt1w: - return true; + distVal = (ssize_t)(dstAddr - srcAddr); - default: - break; - } - break; + if (dstOffs <= srcOffs) + { +#if DEBUG_EMIT + /* This is a backward jump - distance is known at this point */ - case IF_SVE_IP_4A: - switch (ins) - { - case INS_sve_ld1roh: - case INS_sve_ld1row: - case INS_sve_ld1rod: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - case INS_sve_ld1rqd: - return true; + if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) + { + size_t blkOffs = id->idjIG->igOffs; - default: - break; - } - break; + if (INTERESTING_JUMP_NUM == 0) + printf("[3] Jump %u:\n", id->idDebugOnlyInfo()->idNum); + printf("[3] Jump block is at %08X - %02X = %08X\n", blkOffs, emitOffsAdj, blkOffs - emitOffsAdj); + printf("[3] Jump is at %08X - %02X = %08X\n", srcOffs, emitOffsAdj, srcOffs - emitOffsAdj); + printf("[3] Label block is at %08X - %02X = %08X\n", dstOffs, emitOffsAdj, dstOffs - emitOffsAdj); + } +#endif + } + else + { + /* This is a forward jump - distance will be an upper limit */ - case IF_SVE_IR_4A: - switch (ins) - { - case INS_sve_ld2q: - case INS_sve_ld3q: - case INS_sve_ld4q: - return true; + emitFwdJumps = true; - default: - break; - } - break; + /* The target offset will be closer by at least 'emitOffsAdj', but only if this + jump doesn't cross the hot-cold boundary. */ - case IF_SVE_IT_4A: - switch (ins) - { - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - return true; + if (!emitJumpCrossHotColdBoundary(srcOffs, dstOffs)) + { + dstOffs -= emitOffsAdj; + distVal -= emitOffsAdj; + } - default: - break; - } - break; + /* Record the location of the jump for later patching */ - case IF_SVE_IU_4B: - switch (ins) - { - case INS_sve_ld1sw: - case INS_sve_ldff1sw: - case INS_sve_ld1d: - case INS_sve_ldff1d: - return true; + id->idjOffs = dstOffs; - default: - break; - } - break; + /* Are we overflowing the id->idjOffs bitfield? */ + if (id->idjOffs != dstOffs) + IMPL_LIMITATION("Method is too large"); - case IF_SVE_JB_4A: - switch (ins) - { - case INS_sve_stnt1h: - case INS_sve_stnt1w: - case INS_sve_stnt1d: - return true; +#if DEBUG_EMIT + if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) + { + size_t blkOffs = id->idjIG->igOffs; - default: - break; - } - break; + if (INTERESTING_JUMP_NUM == 0) + printf("[4] Jump %u:\n", id->idDebugOnlyInfo()->idNum); + printf("[4] Jump block is at %08X\n", blkOffs); + printf("[4] Jump is at %08X\n", srcOffs); + printf("[4] Label block is at %08X - %02X = %08X\n", dstOffs + emitOffsAdj, emitOffsAdj, dstOffs); + } +#endif + } - case IF_SVE_JC_4A: - switch (ins) - { - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - return true; +#ifdef DEBUG + if (0 && emitComp->verbose) + { + size_t sz = 4; + int distValSize = id->idjShort ? 4 : 8; + printf("; %s jump [%08X/%03u] from %0*X to %0*X: dist = 0x%08X\n", (dstOffs <= srcOffs) ? "Fwd" : "Bwd", + dspPtr(id), id->idDebugOnlyInfo()->idNum, distValSize, srcOffs + sz, distValSize, dstOffs, distVal); + } +#endif - default: - break; - } - break; + /* For forward jumps, record the address of the distance value */ + id->idjTemp.idjAddr = (distVal > 0) ? dst : NULL; - case IF_SVE_JD_4C: - switch (ins) - { - case INS_sve_st1w: - case INS_sve_st1d: - return true; + assert(insOptsNone(id->idInsOpt())); - default: - break; - } - break; + if (isJump) + { + if (id->idjShort) + { + // Short conditional/unconditional jump + assert(!id->idjKeepLong); + assert(emitJumpCrossHotColdBoundary(srcOffs, dstOffs) == false); + assert((fmt == IF_BI_0A) || (fmt == IF_BI_0B) || (fmt == IF_BI_1A) || (fmt == IF_BI_1B)); + dst = emitOutputShortBranch(dst, ins, fmt, distVal, id); + } + else + { + // Long conditional/unconditional jump - case IF_SVE_JD_4C_A: - switch (ins) + if (fmt == IF_LARGEJMP) { - case INS_sve_st1d: - return true; + // This is a pseudo-instruction format representing a large conditional branch, to allow + // us to get a greater branch target range than we can get by using a straightforward conditional + // branch. It is encoded as a short conditional branch that branches around a long unconditional + // branch. + // + // Conceptually, we have: + // + // b L_target + // + // The code we emit is: + // + // b L_not // 4 bytes. Note that we reverse the condition. + // b L_target // 4 bytes + // L_not: + // + // Note that we don't actually insert any blocks: we simply encode "b L_not" as a branch with + // the correct offset. Note also that this works for both integer and floating-point conditions, because + // the condition inversion takes ordered/unordered into account, preserving NaN behavior. For example, + // "GT" (greater than) is inverted to "LE" (less than, equal, or unordered). - default: - break; - } - break; + instruction reverseIns; + insFormat reverseFmt; - case IF_SVE_JF_4A: - switch (ins) - { - case INS_sve_st2q: - case INS_sve_st3q: - case INS_sve_st4q: - return true; + switch (ins) + { + case INS_cbz: + reverseIns = INS_cbnz; + reverseFmt = IF_BI_1A; + break; + case INS_cbnz: + reverseIns = INS_cbz; + reverseFmt = IF_BI_1A; + break; + case INS_tbz: + reverseIns = INS_tbnz; + reverseFmt = IF_BI_1B; + break; + case INS_tbnz: + reverseIns = INS_tbz; + reverseFmt = IF_BI_1B; + break; + default: + reverseIns = emitJumpKindToIns(emitReverseJumpKind(emitInsToJumpKind(ins))); + reverseFmt = IF_BI_0B; + } - default: - break; + dst = emitOutputShortBranch(dst, + reverseIns, // reverse the conditional instruction + reverseFmt, 8, /* 8 bytes from start of this large conditional + pseudo-instruction to L_not. */ + id); + + // Now, pretend we've got a normal unconditional branch, and fall through to the code to emit that. + ins = INS_b; + fmt = IF_BI_0A; + + // The distVal was computed based on the beginning of the pseudo-instruction, + // So subtract the size of the conditional branch so that it is relative to the + // unconditional branch. + distVal -= 4; } - break; - case IF_SVE_JJ_4B: - switch (ins) + assert(fmt == IF_BI_0A); + assert((distVal & 1) == 0); + code_t code = emitInsCode(ins, fmt); + const bool doRecordRelocation = emitComp->opts.compReloc && emitJumpCrossHotColdBoundary(srcOffs, dstOffs); + + if (doRecordRelocation) { - case INS_sve_st1h: - case INS_sve_st1w: - case INS_sve_st1d: - return true; + // dst isn't an actual final target location, just some intermediate + // location. Thus we cannot make any guarantees about distVal (not + // even the direction/sign). Instead we don't encode any offset and + // rely on the relocation to do all the work + } + else + { + // Branch offset encodings are scaled by 4. + noway_assert((distVal & 3) == 0); + distVal >>= 2; + noway_assert(isValidSimm<26>(distVal)); - default: - break; + // Insert offset into unconditional branch instruction + distVal &= 0x3FFFFFFLL; + code |= distVal; } - break; - case IF_SVE_HY_3B: - case IF_SVE_IB_3A: - switch (ins) - { - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - return true; + const unsigned instrSize = emitOutput_Instr(dst, code); - default: - break; + if (doRecordRelocation) + { + assert(id->idjKeepLong); + if (emitComp->info.compMatchedVM) + { + void* target = emitOffsetToPtr(dstOffs); + emitRecordRelocation((void*)dst, target, IMAGE_REL_ARM64_BRANCH26); + } } - break; - default: - break; + dst += instrSize; + } + } + else if (loadLabel) + { + dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); } - return false; + return dst; } /***************************************************************************** - * - * Returns true if the SVE instruction has a addr. - * This is for formats that have [, .T, ], [, .T, #N] - */ -/*static*/ bool emitter::insSveIsModN(instruction ins, insFormat fmt) +* +* Output a short branch instruction. +*/ +BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id) { - switch (fmt) + code_t code = emitInsCode(ins, fmt); + + ssize_t loBits = (distVal & 3); + noway_assert(loBits == 0); + distVal >>= 2; // branch offset encodings are scaled by 4. + + if (fmt == IF_BI_0A) { - case IF_SVE_JJ_4A: - case IF_SVE_JJ_4A_B: - switch (ins) - { - case INS_sve_st1d: - case INS_sve_st1h: - case INS_sve_st1w: - return true; + // INS_b or INS_bl_local + noway_assert(isValidSimm<26>(distVal)); + distVal &= 0x3FFFFFFLL; + code |= distVal; + } + else if (fmt == IF_BI_0B) // BI_0B 01010100iiiiiiii iiiiiiiiiiiXXXXX simm19:00 + { + // INS_beq, INS_bne, etc... + noway_assert(isValidSimm<19>(distVal)); + distVal &= 0x7FFFFLL; + code |= distVal << 5; + } + else if (fmt == IF_BI_1A) // BI_1A X.......iiiiiiii iiiiiiiiiiittttt Rt simm19:00 + { + // INS_cbz or INS_cbnz + assert(id != nullptr); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt - default: - break; - } - break; + noway_assert(isValidSimm<19>(distVal)); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + } + else if (fmt == IF_BI_1B) // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 + { + // INS_tbz or INS_tbnz + assert(id != nullptr); + ssize_t imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - switch (ins) - { - case INS_sve_st1h: - case INS_sve_st1w: - return true; + if (imm & 0x20) // test bit 32-63 ? + { + code |= 0x80000000; // B + } + code |= ((imm & 0x1F) << 19); // bbbbb + code |= insEncodeReg_Rt(id->idReg1()); // ttttt - default: - break; - } - break; + noway_assert(isValidSimm<14>(distVal)); + distVal &= 0x3FFFLL; // 14 bits + code |= distVal << 5; + } + else + { + assert(!"Unknown fmt for emitOutputShortBranch"); + } - case IF_SVE_JK_4A: - case IF_SVE_JK_4A_B: - switch (ins) - { - case INS_sve_st1b: - return true; + dst += emitOutput_Instr(dst, code); - default: - break; - } - break; + return dst; +} - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - switch (ins) - { - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ld1sb: - case INS_sve_ld1sh: - case INS_sve_ld1w: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - case INS_sve_ldff1sb: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; +/***************************************************************************** +* +* Output a short address instruction. +*/ +BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg) +{ + ssize_t loBits = (distVal & 3); + distVal >>= 2; - default: - break; - } - break; + code_t code = emitInsCode(ins, fmt); + if (fmt == IF_DI_1E) // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + { + // INS_adr or INS_adrp + code |= insEncodeReg_Rd(reg); // ddddd - case IF_SVE_HW_4A_B: - case IF_SVE_HW_4A_C: - switch (ins) - { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ld1w: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; + noway_assert(isValidSimm<19>(distVal)); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; + code |= loBits << 29; // 2 bits + } + else + { + assert(!"Unknown fmt for emitOutputShortAddress"); + } - default: - break; - } - break; + dst += emitOutput_Instr(dst, code); - case IF_SVE_IU_4A: - switch (ins) - { - case INS_sve_ld1d: - case INS_sve_ld1sw: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - return true; + return dst; +} - default: - break; - } - break; +/***************************************************************************** +* +* Output a short constant instruction. +*/ +BYTE* emitter::emitOutputShortConstant( + BYTE* dst, instruction ins, insFormat fmt, ssize_t imm, regNumber reg, emitAttr opSize) +{ + code_t code = emitInsCode(ins, fmt); - case IF_SVE_IU_4A_A: - switch (ins) - { - case INS_sve_ld1sw: - case INS_sve_ldff1d: - case INS_sve_ldff1sw: - return true; + if (fmt == IF_LS_1A) + { + // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt simm21 + // INS_ldr or INS_ldrsw (PC-Relative) - default: - break; - } - break; + ssize_t loBits = (imm & 3); + noway_assert(loBits == 0); + ssize_t distVal = imm >> 2; // load offset encodings are scaled by 4. - case IF_SVE_IU_4A_C: - switch (ins) - { - case INS_sve_ld1d: - return true; + noway_assert(isValidSimm<19>(distVal)); - default: - break; - } - break; - - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - switch (ins) - { - case INS_sve_prfb: - case INS_sve_prfh: - case INS_sve_prfw: - case INS_sve_prfd: - return true; - - default: - break; - } - break; - - default: - break; - } - - return false; -} - -/***************************************************************************** - * - * Returns 0, 1, 2, 3 or 4 depending on the instruction and format. - * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], - * [{, , LSL #N}] - */ - -/*static*/ int emitter::insSveGetLslOrModN(instruction ins, insFormat fmt) -{ - switch (fmt) - { - case IF_SVE_JD_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1h: - return 1; - - default: - break; - } - break; - - case IF_SVE_JD_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1w: - return 2; - - default: - break; - } - break; - - case IF_SVE_HW_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - return 1; - - case INS_sve_ld1w: - case INS_sve_ldff1w: - return 2; - - default: - break; - } - break; - - case IF_SVE_JJ_4A: - case IF_SVE_JJ_4A_B: - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - case IF_SVE_JK_4A: - case IF_SVE_JK_4A_B: - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - case IF_SVE_HW_4A_B: - case IF_SVE_HW_4A_C: - case IF_SVE_IU_4A: - case IF_SVE_IU_4A_A: - case IF_SVE_IU_4A_C: - assert(!insSveIsLslN(ins, fmt)); - assert(insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1h: - case INS_sve_ld1sh: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - switch (fmt) - { - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - return 1; - - default: - break; - } - return 0; - - case INS_sve_ld1w: - case INS_sve_ldff1w: - case INS_sve_ld1sw: - case INS_sve_ldff1sw: - switch (fmt) - { - case IF_SVE_HW_4A: - case IF_SVE_HW_4A_A: - case IF_SVE_IU_4A: - return 2; - - default: - break; - } - return 0; - - case INS_sve_ld1d: - case INS_sve_ldff1d: - switch (fmt) - { - case IF_SVE_IU_4A: - return 3; - - default: - break; - } - return 0; - - case INS_sve_st1h: - switch (fmt) - { - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - return 0; - - default: - break; - } - return 1; - - case INS_sve_st1w: - switch (fmt) - { - case IF_SVE_JJ_4A_C: - case IF_SVE_JJ_4A_D: - return 0; - - default: - break; - } - return 2; - - case INS_sve_st1d: - if (fmt == IF_SVE_JJ_4A_B) - { - return 0; - } - return 3; - - default: - break; - } - return 0; - - case IF_SVE_IG_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldff1sw: - return 2; - - case INS_sve_ldff1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_IG_4A_F: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldff1sh: - return 1; - - case INS_sve_ldff1w: - return 2; - - default: - break; - } - break; - - case IF_SVE_IG_4A_G: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldff1h: - return 1; - - default: - break; - } - break; - - case IF_SVE_II_4A: - case IF_SVE_II_4A_B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_II_4A_H: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1w: - return 2; - - default: - break; - } - break; - - case IF_SVE_IK_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1sw: - return 2; - - default: - break; - } - break; - - case IF_SVE_IK_4A_G: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1sh: - return 1; - - default: - break; - } - break; - - case IF_SVE_IK_4A_I: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1h: - return 1; - - default: - break; - } - break; - - case IF_SVE_IN_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ldnt1h: - return 1; - case INS_sve_ldnt1w: - return 2; - case INS_sve_ldnt1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_IP_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1roh: - case INS_sve_ld1rqh: - return 1; - - case INS_sve_ld1row: - case INS_sve_ld1rqw: - return 2; - case INS_sve_ld1rod: - case INS_sve_ld1rqd: - return 3; - - default: - break; - } - break; - - case IF_SVE_IR_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld2q: - case INS_sve_ld3q: - case INS_sve_ld4q: - return 4; - - default: - break; - } - break; - - case IF_SVE_IT_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld2h: - case INS_sve_ld3h: - case INS_sve_ld4h: - return 1; - - case INS_sve_ld2w: - case INS_sve_ld3w: - case INS_sve_ld4w: - return 2; - - case INS_sve_ld2d: - case INS_sve_ld3d: - case INS_sve_ld4d: - return 3; - - default: - break; - } - break; - - case IF_SVE_IU_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_ld1sw: - case INS_sve_ldff1sw: - return 2; - - case INS_sve_ld1d: - case INS_sve_ldff1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_JB_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_stnt1h: - return 1; - - case INS_sve_stnt1w: - return 2; - - case INS_sve_stnt1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_JC_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st2h: - case INS_sve_st3h: - case INS_sve_st4h: - return 1; - - case INS_sve_st2w: - case INS_sve_st3w: - case INS_sve_st4w: - return 2; - - case INS_sve_st2d: - case INS_sve_st3d: - case INS_sve_st4d: - return 3; - - default: - break; - } - break; - - case IF_SVE_JD_4C: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1w: - return 2; - - case INS_sve_st1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_JD_4C_A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_JF_4A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st2q: - case INS_sve_st3q: - case INS_sve_st4q: - return 4; - - default: - break; - } - break; - - case IF_SVE_JJ_4B: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_st1h: - return 1; - - case INS_sve_st1w: - return 2; - - case INS_sve_st1d: - return 3; - - default: - break; - } - break; - - case IF_SVE_HY_3A: - case IF_SVE_HY_3A_A: - assert(!insSveIsLslN(ins, fmt)); - assert(insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_prfb: - return 0; - - case INS_sve_prfh: - return 1; - - case INS_sve_prfw: - return 2; - - case INS_sve_prfd: - return 3; - - default: - break; - } - break; - - case IF_SVE_HY_3B: - case IF_SVE_IB_3A: - assert(insSveIsLslN(ins, fmt)); - assert(!insSveIsModN(ins, fmt)); - switch (ins) - { - case INS_sve_prfh: - return 1; - - case INS_sve_prfw: - return 2; - - case INS_sve_prfd: - return 3; - - default: - break; - } - break; - - default: - break; - } - - assert(!"Unexpected instruction format"); - return 0; -} - -/***************************************************************************** - * - * Returns true if the specified instruction can encode the 'dtype' field. - */ - -/*static*/ bool emitter::canEncodeSveElemsize_dtype(instruction ins) -{ - switch (ins) - { - case INS_sve_ld1w: - case INS_sve_ld1sb: - case INS_sve_ld1b: - case INS_sve_ld1sh: - case INS_sve_ld1h: - case INS_sve_ldnf1sh: - case INS_sve_ldnf1w: - case INS_sve_ldnf1h: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1b: - case INS_sve_ldff1b: - case INS_sve_ldff1sb: - case INS_sve_ldff1h: - case INS_sve_ldff1sh: - case INS_sve_ldff1w: - return true; - - default: - return false; - } -} - -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * for the 'dtype' field. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(instruction ins, emitAttr size, code_t code) -{ - assert(canEncodeSveElemsize_dtype(ins)); - assert(ins != INS_sve_ld1w); - switch (size) - { - case EA_1BYTE: - switch (ins) - { - case INS_sve_ld1b: - case INS_sve_ldnf1b: - case INS_sve_ldff1b: - return code; // By default, the instruction already encodes 8-bit. - - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; - - case EA_2BYTE: - switch (ins) - { - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ldnf1b: - case INS_sve_ldnf1h: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - return code | (1 << 21); // Set bit '21' to 1. - - case INS_sve_ld1sb: - case INS_sve_ldnf1sb: - case INS_sve_ldff1sb: - return code | (1 << 22); // Set bit '22' to 1. - - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; - - case EA_4BYTE: - switch (ins) - { - case INS_sve_ldnf1w: - case INS_sve_ldff1w: - return code; // By default, the instruction already encodes 32-bit. - - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ldnf1b: - case INS_sve_ldnf1h: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - return code | (1 << 22); // Set bit '22' to 1. - - case INS_sve_ld1sb: - case INS_sve_ld1sh: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1sh: - case INS_sve_ldff1sb: - case INS_sve_ldff1sh: - return code | (1 << 21); // Set bit '21' to 1. - - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; - - case EA_8BYTE: - switch (ins) - { - case INS_sve_ldnf1w: - case INS_sve_ldff1w: - return code | (1 << 21); // Set bit '21' to 1. Set bit '15' to 1. - - case INS_sve_ld1b: - case INS_sve_ld1h: - case INS_sve_ldnf1b: - case INS_sve_ldnf1h: - case INS_sve_ldff1b: - case INS_sve_ldff1h: - return (code | (1 << 22)) | (1 << 21); // Set bit '22' and '21' to 1. - - case INS_sve_ld1sb: - case INS_sve_ld1sh: - case INS_sve_ldnf1sb: - case INS_sve_ldnf1sh: - case INS_sve_ldff1sb: - case INS_sve_ldff1sh: - return code; // By default, the instruction already encodes 64-bit. - - default: - assert(!"Invalid instruction for encoding dtype."); - } - return code; - - default: - assert(!"Invalid size for encoding dtype."); - } - - return code; -} - -/***************************************************************************** - * - * Returns the encoding to select the 4/8/16 byte elemsize for the Arm64 Sve vector instruction 'ld1w' - * for the 'dtype' field. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype_ld1w(instruction ins, - insFormat fmt, - emitAttr size, - code_t code) -{ - assert(canEncodeSveElemsize_dtype(ins)); - assert(ins == INS_sve_ld1w); - switch (size) - { - case EA_4BYTE: - switch (fmt) - { - case IF_SVE_IH_3A_F: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for S. - return (code | (1 << 15)) | (1 << 22); // Set bit '22' and '15' to 1. - - case IF_SVE_II_4A_H: - // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for S. - return (code | (1 << 14)) | (1 << 22); // Set bit '22' and '14' to 1. - - default: - break; - } - break; - - case EA_8BYTE: - switch (fmt) - { - case IF_SVE_IH_3A_F: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for D. - return ((code | (1 << 15)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '15' to 1. - - case IF_SVE_II_4A_H: - // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for D. - return ((code | (1 << 14)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '14' to 1. - - default: - break; - } - break; - - case EA_16BYTE: - switch (fmt) - { - case IF_SVE_IH_3A_F: - return code | (1 << 20); // Set bit '20' to 1. - - case IF_SVE_II_4A_H: - // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the - // proper encoding for Q. - return code | (1 << 15); // Set bit '15' to 1. - - default: - break; - } - break; - - default: - assert(!"Invalid size for encoding dtype."); - break; - } - - assert(!"Invalid instruction format"); - return code; -} - -/***************************************************************************** - * - * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction - * for the 'dtypeh' and 'dtypel' fields. - */ - -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtypeh_dtypel(instruction ins, - insFormat fmt, - emitAttr size, - code_t code) -{ - switch (fmt) - { - case IF_SVE_IC_3A_A: - switch (size) - { - case EA_4BYTE: - switch (ins) - { - case INS_sve_ld1rsh: - return code | (1 << 13); // set bit '13' - - case INS_sve_ld1rw: - return code | (1 << 14); // set bit '14' - - default: - break; - } - break; - - case EA_8BYTE: - switch (ins) - { - case INS_sve_ld1rsh: - return code; - - case INS_sve_ld1rw: - return code | (1 << 14) | (1 << 13); // set bits '14' and '13' - - default: - break; - } - break; - - default: - break; - } - break; - - case IF_SVE_IC_3A_B: - switch (size) - { - case EA_2BYTE: - switch (ins) - { - case INS_sve_ld1rh: - return code | (1 << 13); // set bit '13' - - case INS_sve_ld1rsb: - return code | (1 << 24) | (1 << 14); // set bit '24' and '14' - - default: - break; - } - break; - - case EA_4BYTE: - switch (ins) - { - case INS_sve_ld1rh: - return code | (1 << 14); // set bit '14' - - case INS_sve_ld1rsb: - return code | (1 << 24) | (1 << 13); // set bit '24' and '13' - - default: - break; - } - break; - - case EA_8BYTE: - switch (ins) - { - case INS_sve_ld1rh: - return code | (1 << 14) | (1 << 13); // set bits '14' and '13' - - case INS_sve_ld1rsb: - return code | (1 << 24); // set bit '24' - - default: - break; - } - break; - - default: - break; - } - break; - - case IF_SVE_IC_3A_C: - assert(ins == INS_sve_ld1rb); - switch (size) - { - case EA_1BYTE: - return code; - - case EA_2BYTE: - return code | (1 << 13); // set bit '13' - - case EA_4BYTE: - return code | (1 << 14); // set bit '14' - - case EA_8BYTE: - return code | (1 << 14) | (1 << 13); // set bits '14' and '13' - - default: - break; - } - break; - - default: - break; - } - - assert(!"Unexpected instruction format"); - return code; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 9-bits at bit locations '21-16' for high and '12-10' for low. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm9h9l_21_to_16_and_12_to_10(ssize_t imm) -{ - assert(isValidSimm<9>(imm)); - - if (imm < 0) - { - imm = (imm & 0x1FF); - } - - code_t h = (code_t)(imm & 0x1F8) << 13; // encode high 6-bits at locations '21-16' - code_t l = (code_t)((imm & ~0x1F8) & 0x7) << 10; // encode low 3-bits at locations '12-10' - - return (h | l); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 2 as 4-bits at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf2_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 2>(imm))); - return insEncodeSimm<19, 16>(imm / 2); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 3 as 4-bits at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf3_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 3>(imm))); - return insEncodeSimm<19, 16>(imm / 3); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 4 as 4-bits at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf4_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 4>(imm))); - return insEncodeSimm<19, 16>(imm / 4); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 16 as 4-bits at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf16_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 16>(imm))); - return insEncodeSimm<19, 16>(imm / 16); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 32 as 4-bits at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeSimm4_MultipleOf32_19_to_16(ssize_t imm) -{ - assert((isValidSimm_MultipleOf<4, 32>(imm))); - return insEncodeSimm<19, 16>(imm / 32); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 2 as 5-bits at bit locations '20-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf2_20_to_16(ssize_t imm) -{ - assert((isValidUimm_MultipleOf<5, 2>(imm))); - return insEncodeUimm<20, 16>(imm / 2); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 4 as 5-bits at bit locations '20-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf4_20_to_16(ssize_t imm) -{ - assert((isValidUimm_MultipleOf<5, 4>(imm))); - return insEncodeUimm<20, 16>(imm / 4); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 8 as 5-bits at bit locations '20-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm5_MultipleOf8_20_to_16(ssize_t imm) -{ - assert((isValidUimm_MultipleOf<5, 8>(imm))); - return insEncodeUimm<20, 16>(imm / 8); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 2 as 6-bits at bit locations '21-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf2_21_to_16(ssize_t imm) -{ - assert((isValidUimm_MultipleOf<6, 2>(imm))); - return insEncodeUimm<21, 16>(imm / 2); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 4 as 6-bits at bit locations '21-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf4_21_to_16(ssize_t imm) -{ - assert((isValidUimm_MultipleOf<6, 4>(imm))); - return insEncodeUimm<21, 16>(imm / 4); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value that is a multiple of 8 as 6-bits at bit locations '21-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm6_MultipleOf8_21_to_16(ssize_t imm) -{ - assert((isValidUimm_MultipleOf<6, 8>(imm))); - return insEncodeUimm<21, 16>(imm / 8); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 1-bit at bit locations '23'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm1_23(ssize_t imm) -{ - assert(isValidUimm<1>(imm)); - return (code_t)imm << 23; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 3-bits at bit locations '23-22' for high and '12' for low. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm3h3l_23_to_22_and_12(ssize_t imm) -{ - assert(isValidUimm<3>(imm)); - - code_t h = (code_t)(imm & 0x6) << 21; // encode high 2-bits at locations '23-22' - code_t l = (code_t)(imm & 0x1) << 12; // encode low 1-bit at locations '12' - - return (h | l); -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 4-bits starting from 1, at bit locations '19-16'. - */ - -/*static*/ emitter::code_t emitter::insEncodeUimm4From1_19_to_16(ssize_t imm) -{ - assert(isValidUimmFrom1<4>(imm)); - return (code_t)(imm - 1) << 16; -} - -/***************************************************************************** - * - * Returns the encoding for the immediate value as 8-bits at bit locations '12-5'. - */ - -/*static*/ emitter::code_t emitter::insEncodeImm8_12_to_5(ssize_t imm) -{ - assert(isValidSimm<8>(imm) || isValidUimm<8>(imm)); - return (code_t)((imm & 0xFF) << 5); -} - -/***************************************************************************** - * - * Returns the encoding to select the 4/8-byte width specifier - * at bit location 22 for an Arm64 Sve instruction. - */ -/*static*/ emitter::code_t emitter::insEncodeSveElemsize_R_22(emitAttr size) -{ - if (size == EA_8BYTE) - { - return 0x400000; // set the bit at location 22 - } - - assert(size == EA_4BYTE); - return 0; -} - -/***************************************************************************** - * - * Returns the immediate value for instructions that encode it as a difference - * from tszh:tszl:imm3. - */ -/*static*/ ssize_t emitter::insGetImmDiff(const ssize_t imm, const insOpts opt) -{ - switch (opt) - { - case INS_OPTS_SCALABLE_B: - assert(isValidUimmFrom1<3>(imm)); - return (8 - imm); - - case INS_OPTS_SCALABLE_H: - assert(isValidUimmFrom1<4>(imm)); - return (16 - imm); - - case INS_OPTS_SCALABLE_S: - assert(isValidUimmFrom1<5>(imm)); - return (32 - imm); - - case INS_OPTS_SCALABLE_D: - assert(isValidUimmFrom1<6>(imm)); - return (64 - imm); - - default: - unreached(); - break; - } - - return 0; -} - -/***************************************************************************** - * - * Returns the two 5-bit signed immediates encoded in the following format: - * njjj jjmi iiii - * - iiiii: the absolute value of imm1 - * - m: 1 if imm1 is negative, 0 otherwise - * - jjjjj: the absolute value of imm2 - * - n: 1 if imm2 is negative, 0 otherwise - */ -/*static*/ ssize_t emitter::insEncodeTwoSimm5(ssize_t imm1, ssize_t imm2) -{ - assert(isValidSimm<5>(imm1)); - assert(isValidSimm<5>(imm2)); - ssize_t immOut = 0; - - if (imm1 < 0) - { - // Set bit location 5 to indicate imm1 is negative - immOut |= 0x20; - imm1 *= -1; - } - - if (imm2 < 0) - { - // Set bit location 11 to indicate imm2 is negative - immOut |= 0x800; - imm2 *= -1; - } - - immOut |= imm1; - immOut |= (imm2 << 6); - return immOut; -} - -/***************************************************************************** - * - * Decodes imm into two 5-bit signed immediates, - * using the encoding format from insEncodeTwoSimm5. - */ -/*static*/ void emitter::insDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2) -{ - assert(imm1 != nullptr); - assert(imm2 != nullptr); - - *imm1 = (imm & 0x1F); - - if ((imm & 0x20) != 0) - { - *imm1 *= -1; - } - - imm >>= 6; - *imm2 = (imm & 0x1F); - - if ((imm & 0x20) != 0) - { - *imm2 *= -1; - } - - assert(isValidSimm<5>(*imm1)); - assert(isValidSimm<5>(*imm2)); -} - -/***************************************************************************** - * - * Returns the encoding to select an insSvePattern - */ -/*static*/ emitter::code_t emitter::insEncodeSvePattern(insSvePattern pattern) -{ - return (code_t)((unsigned)pattern << 5); -} - -BYTE* emitter::emitOutputLoadLabel(BYTE* dst, BYTE* srcAddr, BYTE* dstAddr, instrDescJmp* id) -{ - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - regNumber dstReg = id->idReg1(); - if (id->idjShort) - { - // adr x, [rel addr] -- compute address: current addr(ip) + rel addr. - assert(ins == INS_adr); - assert(fmt == IF_DI_1E); - ssize_t distVal = (ssize_t)(dstAddr - srcAddr); - dst = emitOutputShortAddress(dst, ins, fmt, distVal, dstReg); - } - else - { - // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr - assert(fmt == IF_LARGEADR); - ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); - dst = emitOutputShortAddress(dst, INS_adrp, IF_DI_1E, relPageAddr, dstReg); - - // add x, x, page offs -- compute address = page addr + page offs - ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits - assert(isValidUimm<12>(imm12)); - code_t code = - emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) - code |= insEncodeDatasize(EA_8BYTE); // X - code |= ((code_t)imm12 << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rd(dstReg); // ddddd - code |= insEncodeReg_Rn(dstReg); // nnnnn - dst += emitOutput_Instr(dst, code); - } - return dst; -} - -/***************************************************************************** - * - * Output a local jump or other instruction with a pc-relative immediate. - * Note that this may be invoked to overwrite an existing jump instruction at 'dst' - * to handle forward branch patching. - */ - -BYTE* emitter::emitOutputLJ(insGroup* ig, BYTE* dst, instrDesc* i) -{ - instrDescJmp* id = (instrDescJmp*)i; - - unsigned srcOffs; - unsigned dstOffs; - BYTE* srcAddr; - BYTE* dstAddr; - ssize_t distVal; - - // Set default ins/fmt from id. - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - - bool loadLabel = false; - bool isJump = false; - bool loadConstant = false; - - switch (ins) - { - default: - isJump = true; - break; - - case INS_tbz: - case INS_tbnz: - case INS_cbz: - case INS_cbnz: - isJump = true; - break; - - case INS_ldr: - case INS_ldrsw: - loadConstant = true; - break; - - case INS_adr: - case INS_adrp: - loadLabel = true; - break; - } - - /* Figure out the distance to the target */ - - srcOffs = emitCurCodeOffs(dst); - srcAddr = emitOffsetToPtr(srcOffs); - - if (id->idAddr()->iiaIsJitDataOffset()) - { - assert(loadConstant || loadLabel); - int doff = id->idAddr()->iiaGetJitDataOffset(); - assert(doff >= 0); - ssize_t imm = emitGetInsSC(id); - assert((imm >= 0) && (imm < 0x1000)); // 0x1000 is arbitrary, currently 'imm' is always 0 - - unsigned dataOffs = (unsigned)(doff + imm); - assert(dataOffs < emitDataSize()); - dstAddr = emitDataOffsetToPtr(dataOffs); - - regNumber dstReg = id->idReg1(); - regNumber addrReg = dstReg; // an integer register to compute long address. - emitAttr opSize = id->idOpSize(); - - if (loadConstant) - { - if (id->idjShort) - { - // ldr x/v, [rel addr] -- load constant from current addr(ip) + rel addr. - assert(ins == INS_ldr); - assert(fmt == IF_LS_1A); - distVal = (ssize_t)(dstAddr - srcAddr); - dst = emitOutputShortConstant(dst, ins, fmt, distVal, dstReg, opSize); - } - else - { - // adrp x, [rel page addr] -- compute page address: current page addr + rel page addr - assert(fmt == IF_LARGELDC); - ssize_t relPageAddr = computeRelPageAddr((size_t)dstAddr, (size_t)srcAddr); - if (isVectorRegister(dstReg)) - { - // Update addrReg with the reserved integer register - // since we cannot use dstReg (vector) to load constant directly from memory. - - // If loading a 16-byte value, we will need to load directly into dstReg. - // Thus, encode addrReg for the ld1 instruction. - if (opSize == EA_16BYTE) - { - addrReg = encodingSPtoZR(id->idReg2()); - } - else - { - addrReg = id->idReg2(); - } - - assert(isGeneralRegister(addrReg)); - } - - ins = INS_adrp; - fmt = IF_DI_1E; - dst = emitOutputShortAddress(dst, ins, fmt, relPageAddr, addrReg); - - ssize_t imm12 = (ssize_t)dstAddr & 0xFFF; // 12 bits - assert(isValidUimm<12>(imm12)); - - // Special case: emit add + ld1 instructions for loading 16-byte data into vector register. - if (isVectorRegister(dstReg) && (opSize == EA_16BYTE)) - { - const emitAttr elemSize = EA_1BYTE; - const insOpts opt = optMakeArrangement(opSize, elemSize); - - assert(isGeneralRegisterOrSP(addrReg)); - assert(isValidVectorElemsize(elemSize)); - assert(isValidArrangement(opSize, opt)); - - // Calculate page addr + page offs, then emit ld1 instruction. - dst = emitOutputVectorConstant(dst, imm12, dstReg, addrReg, opSize, elemSize); - } - else - { - // ldr x, [x, 0] -- load constant from address into integer register. - ins = INS_ldr; - fmt = IF_LS_2B; - dst = emitOutputShortConstant(dst, ins, fmt, imm12, addrReg, opSize); - - // fmov v, d -- copy constant in integer register to vector register. - // This is needed only for vector constant. - if (addrReg != dstReg) - { - // fmov Vd,Rn DV_2I X00111100X100111 000000nnnnnddddd 1E27 0000 Vd,Rn - // (scalar, from general) - assert(isVectorRegister(dstReg) && isGeneralRegister(addrReg)); - ins = INS_fmov; - fmt = IF_DV_2I; - code_t code = emitInsCode(ins, fmt); - - code |= insEncodeReg_Vd(dstReg); // ddddd - code |= insEncodeReg_Rn(addrReg); // nnnnn - if (id->idOpSize() == EA_8BYTE) - { - code |= 0x80400000; // X ... X - } - dst += emitOutput_Instr(dst, code); - } - } - } - } - else - { - assert(loadLabel); - dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); - } - - return dst; - } - - assert(loadLabel || isJump); - - if (id->idAddr()->iiaHasInstrCount()) - { - assert(ig != NULL); - int instrCount = id->idAddr()->iiaGetInstrCount(); - unsigned insNum = emitFindInsNum(ig, id); - if (instrCount < 0) - { - // Backward branches using instruction count must be within the same instruction group. - assert(insNum + 1 >= (unsigned)(-instrCount)); - } - - dstOffs = ig->igOffs + emitFindOffset(ig, (insNum + 1 + instrCount)); - dstAddr = emitOffsetToPtr(dstOffs); - } - else - { - dstOffs = id->idAddr()->iiaIGlabel->igOffs; - dstAddr = emitOffsetToPtr(dstOffs); - } - - distVal = (ssize_t)(dstAddr - srcAddr); - - if (dstOffs <= srcOffs) - { -#if DEBUG_EMIT - /* This is a backward jump - distance is known at this point */ - - if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) - { - size_t blkOffs = id->idjIG->igOffs; - - if (INTERESTING_JUMP_NUM == 0) - printf("[3] Jump %u:\n", id->idDebugOnlyInfo()->idNum); - printf("[3] Jump block is at %08X - %02X = %08X\n", blkOffs, emitOffsAdj, blkOffs - emitOffsAdj); - printf("[3] Jump is at %08X - %02X = %08X\n", srcOffs, emitOffsAdj, srcOffs - emitOffsAdj); - printf("[3] Label block is at %08X - %02X = %08X\n", dstOffs, emitOffsAdj, dstOffs - emitOffsAdj); - } -#endif - } - else - { - /* This is a forward jump - distance will be an upper limit */ - - emitFwdJumps = true; - - /* The target offset will be closer by at least 'emitOffsAdj', but only if this - jump doesn't cross the hot-cold boundary. */ - - if (!emitJumpCrossHotColdBoundary(srcOffs, dstOffs)) - { - dstOffs -= emitOffsAdj; - distVal -= emitOffsAdj; - } - - /* Record the location of the jump for later patching */ - - id->idjOffs = dstOffs; - - /* Are we overflowing the id->idjOffs bitfield? */ - if (id->idjOffs != dstOffs) - IMPL_LIMITATION("Method is too large"); - -#if DEBUG_EMIT - if (id->idDebugOnlyInfo()->idNum == (unsigned)INTERESTING_JUMP_NUM || INTERESTING_JUMP_NUM == 0) - { - size_t blkOffs = id->idjIG->igOffs; - - if (INTERESTING_JUMP_NUM == 0) - printf("[4] Jump %u:\n", id->idDebugOnlyInfo()->idNum); - printf("[4] Jump block is at %08X\n", blkOffs); - printf("[4] Jump is at %08X\n", srcOffs); - printf("[4] Label block is at %08X - %02X = %08X\n", dstOffs + emitOffsAdj, emitOffsAdj, dstOffs); - } -#endif - } - -#ifdef DEBUG - if (0 && emitComp->verbose) - { - size_t sz = 4; - int distValSize = id->idjShort ? 4 : 8; - printf("; %s jump [%08X/%03u] from %0*X to %0*X: dist = 0x%08X\n", (dstOffs <= srcOffs) ? "Fwd" : "Bwd", - dspPtr(id), id->idDebugOnlyInfo()->idNum, distValSize, srcOffs + sz, distValSize, dstOffs, distVal); - } -#endif - - /* For forward jumps, record the address of the distance value */ - id->idjTemp.idjAddr = (distVal > 0) ? dst : NULL; - - assert(insOptsNone(id->idInsOpt())); - - if (isJump) - { - if (id->idjShort) - { - // Short conditional/unconditional jump - assert(!id->idjKeepLong); - assert(emitJumpCrossHotColdBoundary(srcOffs, dstOffs) == false); - assert((fmt == IF_BI_0A) || (fmt == IF_BI_0B) || (fmt == IF_BI_1A) || (fmt == IF_BI_1B)); - dst = emitOutputShortBranch(dst, ins, fmt, distVal, id); - } - else - { - // Long conditional/unconditional jump - - if (fmt == IF_LARGEJMP) - { - // This is a pseudo-instruction format representing a large conditional branch, to allow - // us to get a greater branch target range than we can get by using a straightforward conditional - // branch. It is encoded as a short conditional branch that branches around a long unconditional - // branch. - // - // Conceptually, we have: - // - // b L_target - // - // The code we emit is: - // - // b L_not // 4 bytes. Note that we reverse the condition. - // b L_target // 4 bytes - // L_not: - // - // Note that we don't actually insert any blocks: we simply encode "b L_not" as a branch with - // the correct offset. Note also that this works for both integer and floating-point conditions, because - // the condition inversion takes ordered/unordered into account, preserving NaN behavior. For example, - // "GT" (greater than) is inverted to "LE" (less than, equal, or unordered). - - instruction reverseIns; - insFormat reverseFmt; - - switch (ins) - { - case INS_cbz: - reverseIns = INS_cbnz; - reverseFmt = IF_BI_1A; - break; - case INS_cbnz: - reverseIns = INS_cbz; - reverseFmt = IF_BI_1A; - break; - case INS_tbz: - reverseIns = INS_tbnz; - reverseFmt = IF_BI_1B; - break; - case INS_tbnz: - reverseIns = INS_tbz; - reverseFmt = IF_BI_1B; - break; - default: - reverseIns = emitJumpKindToIns(emitReverseJumpKind(emitInsToJumpKind(ins))); - reverseFmt = IF_BI_0B; - } - - dst = emitOutputShortBranch(dst, - reverseIns, // reverse the conditional instruction - reverseFmt, 8, /* 8 bytes from start of this large conditional - pseudo-instruction to L_not. */ - id); - - // Now, pretend we've got a normal unconditional branch, and fall through to the code to emit that. - ins = INS_b; - fmt = IF_BI_0A; - - // The distVal was computed based on the beginning of the pseudo-instruction, - // So subtract the size of the conditional branch so that it is relative to the - // unconditional branch. - distVal -= 4; - } - - assert(fmt == IF_BI_0A); - assert((distVal & 1) == 0); - code_t code = emitInsCode(ins, fmt); - const bool doRecordRelocation = emitComp->opts.compReloc && emitJumpCrossHotColdBoundary(srcOffs, dstOffs); - - if (doRecordRelocation) - { - // dst isn't an actual final target location, just some intermediate - // location. Thus we cannot make any guarantees about distVal (not - // even the direction/sign). Instead we don't encode any offset and - // rely on the relocation to do all the work - } - else - { - // Branch offset encodings are scaled by 4. - noway_assert((distVal & 3) == 0); - distVal >>= 2; - noway_assert(isValidSimm<26>(distVal)); - - // Insert offset into unconditional branch instruction - distVal &= 0x3FFFFFFLL; - code |= distVal; - } - - const unsigned instrSize = emitOutput_Instr(dst, code); - - if (doRecordRelocation) - { - assert(id->idjKeepLong); - if (emitComp->info.compMatchedVM) - { - void* target = emitOffsetToPtr(dstOffs); - emitRecordRelocation((void*)dst, target, IMAGE_REL_ARM64_BRANCH26); - } - } - - dst += instrSize; - } - } - else if (loadLabel) - { - dst = emitOutputLoadLabel(dst, srcAddr, dstAddr, id); - } - - return dst; -} - -/***************************************************************************** -* -* Output a short branch instruction. -*/ -BYTE* emitter::emitOutputShortBranch(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, instrDescJmp* id) -{ - code_t code = emitInsCode(ins, fmt); - - ssize_t loBits = (distVal & 3); - noway_assert(loBits == 0); - distVal >>= 2; // branch offset encodings are scaled by 4. - - if (fmt == IF_BI_0A) - { - // INS_b or INS_bl_local - noway_assert(isValidSimm<26>(distVal)); - distVal &= 0x3FFFFFFLL; - code |= distVal; - } - else if (fmt == IF_BI_0B) // BI_0B 01010100iiiiiiii iiiiiiiiiiiXXXXX simm19:00 - { - // INS_beq, INS_bne, etc... - noway_assert(isValidSimm<19>(distVal)); - distVal &= 0x7FFFFLL; - code |= distVal << 5; - } - else if (fmt == IF_BI_1A) // BI_1A X.......iiiiiiii iiiiiiiiiiittttt Rt simm19:00 - { - // INS_cbz or INS_cbnz - assert(id != nullptr); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - - noway_assert(isValidSimm<19>(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; - } - else if (fmt == IF_BI_1B) // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 - { - // INS_tbz or INS_tbnz - assert(id != nullptr); - ssize_t imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - - if (imm & 0x20) // test bit 32-63 ? - { - code |= 0x80000000; // B - } - code |= ((imm & 0x1F) << 19); // bbbbb - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - - noway_assert(isValidSimm<14>(distVal)); - distVal &= 0x3FFFLL; // 14 bits - code |= distVal << 5; - } - else - { - assert(!"Unknown fmt for emitOutputShortBranch"); - } - - dst += emitOutput_Instr(dst, code); - - return dst; -} - -/***************************************************************************** -* -* Output a short address instruction. -*/ -BYTE* emitter::emitOutputShortAddress(BYTE* dst, instruction ins, insFormat fmt, ssize_t distVal, regNumber reg) -{ - ssize_t loBits = (distVal & 3); - distVal >>= 2; - - code_t code = emitInsCode(ins, fmt); - if (fmt == IF_DI_1E) // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 - { - // INS_adr or INS_adrp - code |= insEncodeReg_Rd(reg); // ddddd - - noway_assert(isValidSimm<19>(distVal)); - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; - code |= loBits << 29; // 2 bits - } - else - { - assert(!"Unknown fmt for emitOutputShortAddress"); - } - - dst += emitOutput_Instr(dst, code); - - return dst; -} - -/***************************************************************************** -* -* Output a short constant instruction. -*/ -BYTE* emitter::emitOutputShortConstant( - BYTE* dst, instruction ins, insFormat fmt, ssize_t imm, regNumber reg, emitAttr opSize) -{ - code_t code = emitInsCode(ins, fmt); - - if (fmt == IF_LS_1A) - { - // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt simm21 - // INS_ldr or INS_ldrsw (PC-Relative) - - ssize_t loBits = (imm & 3); - noway_assert(loBits == 0); - ssize_t distVal = imm >> 2; // load offset encodings are scaled by 4. - - noway_assert(isValidSimm<19>(distVal)); - - // Is the target a vector register? - if (isVectorRegister(reg)) - { - code |= insEncodeDatasizeVLS(code, opSize); // XX V - code |= insEncodeReg_Vt(reg); // ttttt - } - else - { - assert(isGeneralRegister(reg)); - // insEncodeDatasizeLS is not quite right for this case. - // So just specialize it. - if ((ins == INS_ldr) && (opSize == EA_8BYTE)) - { - // set the operation size in bit 30 - code |= 0x40000000; - } - - code |= insEncodeReg_Rt(reg); // ttttt - } - - distVal &= 0x7FFFFLL; // 19 bits - code |= distVal << 5; - } - else if (fmt == IF_LS_2B) - { - // ldr Rt,[Xn+pimm12] LS_2B 1X11100101iiiiii iiiiiinnnnnttttt B940 0000 imm(0-4095<<{2,3}) - // INS_ldr or INS_ldrsw (PC-Relative) - noway_assert(isValidUimm<12>(imm)); - assert(isGeneralRegister(reg)); - - if (opSize == EA_8BYTE) - { - // insEncodeDatasizeLS is not quite right for this case. - // So just specialize it. - if (ins == INS_ldr) - { - // set the operation size in bit 30 - code |= 0x40000000; - } - // Low 3 bits should be 0 -- 8 byte JIT data should be aligned on 8 byte. - assert((imm & 7) == 0); - imm >>= 3; - } - else - { - assert(opSize == EA_4BYTE); - // Low 2 bits should be 0 -- 4 byte aligned data. - assert((imm & 3) == 0); - imm >>= 2; - } - - code |= insEncodeReg_Rt(reg); // ttttt - code |= insEncodeReg_Rn(reg); // nnnnn - code |= imm << 10; - } - else - { - assert(!"Unknown fmt for emitOutputShortConstant"); - } - - dst += emitOutput_Instr(dst, code); - - return dst; -} - -/***************************************************************************** - * - * Output instructions to load a constant into a vector register. - */ -BYTE* emitter::emitOutputVectorConstant( - BYTE* dst, ssize_t imm, regNumber dstReg, regNumber addrReg, emitAttr opSize, emitAttr elemSize) -{ - // add addrReg, addrReg, page offs -- compute address = page addr + page offs. - code_t code = emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) - code |= insEncodeDatasize(EA_8BYTE); // X - use EA_8BYTE, as we are calculating 64-bit address - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rd(addrReg); // ddddd - code |= insEncodeReg_Rn(addrReg); // nnnnn - dst += emitOutput_Instr(dst, code); - - // ld1 dstReg, addrReg -- load constant at address in addrReg into dstReg. - code = emitInsCode(INS_ld1, IF_LS_2D); // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn - code |= insEncodeVectorsize(opSize); // Q - code |= insEncodeVLSElemsize(elemSize); // ss - code |= insEncodeReg_Rn(addrReg); // nnnnn - code |= insEncodeReg_Vt(dstReg); // ttttt - dst += emitOutput_Instr(dst, code); - - return dst; -} - -/***************************************************************************** - * - * Output a call instruction. - */ - -unsigned emitter::emitOutputCall(insGroup* ig, BYTE* dst, instrDesc* id, code_t code) -{ - const unsigned char callInstrSize = sizeof(code_t); // 4 bytes - regMaskTP gcrefRegs; - regMaskTP byrefRegs; - - VARSET_TP GCvars(VarSetOps::UninitVal()); - - // Is this a "fat" call descriptor? - if (id->idIsLargeCall()) - { - instrDescCGCA* idCall = (instrDescCGCA*)id; - gcrefRegs = idCall->idcGcrefRegs; - byrefRegs = idCall->idcByrefRegs; - VarSetOps::Assign(emitComp, GCvars, idCall->idcGCvars); - } - else - { - assert(!id->idIsLargeDsp()); - assert(!id->idIsLargeCns()); - - gcrefRegs = emitDecodeCallGCregs(id); - byrefRegs = 0; - VarSetOps::AssignNoCopy(emitComp, GCvars, VarSetOps::MakeEmpty(emitComp)); - } - - /* We update the GC info before the call as the variables cannot be - used by the call. Killing variables before the call helps with - boundary conditions if the call is CORINFO_HELP_THROW - see bug 50029. - If we ever track aliased variables (which could be used by the - call), we would have to keep them alive past the call. */ - - emitUpdateLiveGCvars(GCvars, dst); - -#ifdef DEBUG - // Output any delta in GC variable info, corresponding to the before-call GC var updates done above. - if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) - { - emitDispGCVarDelta(); - } -#endif // DEBUG - - // Now output the call instruction and update the 'dst' pointer - // - unsigned outputInstrSize = emitOutput_Instr(dst, code); - dst += outputInstrSize; - - // All call instructions are 4-byte in size on ARM64 - // - assert(outputInstrSize == callInstrSize); - - // If the method returns a GC ref, mark INTRET (R0) appropriately. - if (id->idGCref() == GCT_GCREF) - { - gcrefRegs |= RBM_INTRET; - } - else if (id->idGCref() == GCT_BYREF) - { - byrefRegs |= RBM_INTRET; - } - - // If is a multi-register return method is called, mark INTRET_1 (X1) appropriately - if (id->idIsLargeCall()) - { - instrDescCGCA* idCall = (instrDescCGCA*)id; - if (idCall->idSecondGCref() == GCT_GCREF) - { - gcrefRegs |= RBM_INTRET_1; - } - else if (idCall->idSecondGCref() == GCT_BYREF) - { - byrefRegs |= RBM_INTRET_1; - } - } - - // If the GC register set has changed, report the new set. - if (gcrefRegs != emitThisGCrefRegs) - { - emitUpdateLiveGCregs(GCT_GCREF, gcrefRegs, dst); - } - // If the Byref register set has changed, report the new set. - if (byrefRegs != emitThisByrefRegs) - { - emitUpdateLiveGCregs(GCT_BYREF, byrefRegs, dst); - } - - // Some helper calls may be marked as not requiring GC info to be recorded. - if ((!id->idIsNoGC())) - { - // On ARM64, as on AMD64, we don't change the stack pointer to push/pop args. - // So we're not really doing a "stack pop" here (note that "args" is 0), but we use this mechanism - // to record the call for GC info purposes. (It might be best to use an alternate call, - // and protect "emitStackPop" under the EMIT_TRACK_STACK_DEPTH preprocessor variable.) - emitStackPop(dst, /*isCall*/ true, callInstrSize, /*args*/ 0); - - // Do we need to record a call location for GC purposes? - // - if (!emitFullGCinfo) - { - emitRecordGCcall(dst, callInstrSize); - } - } - return callInstrSize; -} - -/***************************************************************************** - * - * Emit a 32-bit Arm64 instruction - */ - -unsigned emitter::emitOutput_Instr(BYTE* dst, code_t code) -{ - assert(sizeof(code_t) == 4); - BYTE* dstRW = dst + writeableOffset; - *((code_t*)dstRW) = code; - - return sizeof(code_t); -} - -/***************************************************************************** -* - * Append the machine code corresponding to the given instruction descriptor - * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' - * is the instruction group that contains the instruction. Updates '*dp' to - * point past the generated code, and returns the size of the instruction - * descriptor in bytes. - */ - -size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) -{ - BYTE* dst = *dp; - BYTE* odst = dst; - code_t code = 0; - size_t sz = emitGetInstrDescSize(id); // TODO-ARM64-Cleanup: on ARM, this is set in each case. why? - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - emitAttr size = id->idOpSize(); - -#ifdef DEBUG -#if DUMP_GC_TABLES - bool dspOffs = emitComp->opts.dspGCtbls; -#else - bool dspOffs = !emitComp->opts.disDiffable; -#endif -#endif // DEBUG - - assert(REG_NA == (int)REG_NA); - - /* What instruction format have we got? */ - - switch (fmt) - { - ssize_t imm; - ssize_t index; - ssize_t index2; - unsigned cmode; - unsigned immShift; - emitAttr elemsize; - emitAttr datasize; - - case IF_BI_0A: // BI_0A ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 - case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiii..... simm19:00 - case IF_LARGEJMP: - assert(id->idGCref() == GCT_NONE); - assert(id->idIsBound()); - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); - break; - - case IF_BI_0C: // BI_0C ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 - code = emitInsCode(ins, fmt); - sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); - dst += emitOutputCall(ig, dst, id, code); - // Always call RecordRelocation so that we wire in a JumpStub when we don't reach - emitRecordRelocation(odst, id->idAddr()->iiaAddr, IMAGE_REL_ARM64_BRANCH26); - break; - - case IF_BI_1A: // BI_1A ......iiiiiiiiii iiiiiiiiiiittttt Rt simm19:00 - assert(insOptsNone(id->idInsOpt())); - assert(id->idIsBound()); - - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); - break; - - case IF_BI_1B: // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 - assert(insOptsNone(id->idInsOpt())); - assert(id->idIsBound()); - - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); - break; - - case IF_BR_1A: // BR_1A ................ ......nnnnn..... Rn - assert(insOptsNone(id->idInsOpt())); - assert((ins == INS_ret) || (ins == INS_br)); - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - - dst += emitOutput_Instr(dst, code); - break; - - case IF_BR_1B: // BR_1B ................ ......nnnnn..... Rn - assert(insOptsNone(id->idInsOpt())); - assert((ins == INS_br_tail) || (ins == INS_blr)); - code = emitInsCode(ins, fmt); - - if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && id->idIsTlsGD()) - { - emitRecordRelocation(odst, (CORINFO_METHOD_HANDLE)id->idAddr()->iiaAddr, - IMAGE_REL_AARCH64_TLSDESC_CALL); - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - } - else - { - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - } - dst += emitOutputCall(ig, dst, id, code); - sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); - break; - - case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) - case IF_LARGELDC: - assert(insOptsNone(id->idInsOpt())); - assert(id->idIsBound()); - - dst = emitOutputLJ(ig, dst, id); - sz = sizeof(instrDescJmp); - break; - - case IF_LS_2A: // LS_2A .X.......X...... ......nnnnnttttt Rt Rn - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - if (id->idIsTlsGD()) - { - emitRecordRelocation(odst, (void*)emitGetInsSC(id), IMAGE_REL_AARCH64_TLSDESC_LD64_LO12); - } - break; - - case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) - assert(insOptsNone(id->idInsOpt())); - imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_2C: // LS_2C .X.......X.iiiii iiiiPPnnnnnttttt Rt Rn imm(-256..+255) no/pre/post inc - assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); - imm = emitGetInsSC(id); - assert((imm >= -256) && (imm <= 255)); // signed 9 bits - imm &= 0x1ff; // force into unsigned 9 bit representation - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= insEncodeIndexedOpt(id->idInsOpt()); // PP - code |= ((code_t)imm << 12); // iiiiiiiii - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_2D: // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn - case IF_LS_2E: // LS_2E .Q.............. ....ssnnnnnttttt Vt Rn - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeVLSElemsize(elemsize); // ss - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_2F: // LS_2F .Q.............. xx.Sssnnnnnttttt Vt[] Rn - case IF_LS_2G: // LS_2G .Q.............. xx.Sssnnnnnttttt Vt[] Rn - elemsize = id->idOpSize(); - index = id->idSmallCns(); - code = emitInsCode(ins, fmt); - - code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_3A: // LS_3A .X.......X.mmmmm oooS..nnnnnttttt Rt Rn Rm ext(Rm) LSL {} - assert(insOptsLSExtend(id->idInsOpt())); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - } - else - { - code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - } - code |= insEncodeExtend(id->idInsOpt()); // ooo - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - if (id->idIsLclVar()) - { - code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm - } - else - { - code |= insEncodeReg3Scale(id->idReg3Scaled()); // S - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - } - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_3B: // LS_3B X............... .aaaaannnnnddddd Rd Ra Rn - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - code |= insEncodeReg_Va(id->idReg2()); // aaaaa - } - else - { - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - code |= insEncodeReg_Ra(id->idReg2()); // aaaaa - } - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_3C: // LS_3C X......PP.iiiiii iaaaaannnnnddddd Rd Ra Rn imm(im7,sh) - assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); - imm = emitGetInsSC(id); - assert((imm >= -64) && (imm <= 63)); // signed 7 bits - imm &= 0x7f; // force into unsigned 7 bit representation - code = emitInsCode(ins, fmt); - // Is the target a vector register? - if (isVectorRegister(id->idReg1())) - { - code &= 0x3FFFFFFF; // clear the size bits - code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - code |= insEncodeReg_Va(id->idReg2()); // aaaaa - } - else - { - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - code |= insEncodeReg_Ra(id->idReg2()); // aaaaa - } - code |= insEncodePairIndexedOpt(ins, id->idInsOpt()); // PP - code |= ((code_t)imm << 15); // iiiiiiiii - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_3D: // LS_3D .X.......X.mmmmm ......nnnnnttttt Wm Rt Rn - code = emitInsCode(ins, fmt); - // Arm64 store exclusive unpredictable cases - assert(id->idReg1() != id->idReg2()); - assert(id->idReg1() != id->idReg3()); - code |= insEncodeDatasizeLS(code, id->idOpSize()); // X - code |= insEncodeReg_Rm(id->idReg1()); // mmmmm - code |= insEncodeReg_Rt(id->idReg2()); // ttttt - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_3E: // LS_3E .X.........mmmmm ......nnnnnttttt Rm Rt Rn ARMv8.1 LSE Atomics - code = emitInsCode(ins, fmt); - code |= insEncodeDatasizeLS(code, id->idOpSize()); // X - code |= insEncodeReg_Rm(id->idReg1()); // mmmmm - code |= insEncodeReg_Rt(id->idReg2()); // ttttt - code |= insEncodeReg_Rn(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - - // Some instructions with this encoding return their result in the - // second operand register instead of the first so we special case - // the GC update here and skip the common path down below. - if (emitInsDestIsOp2(ins)) - { - if (id->idGCref() != GCT_NONE) - { - emitGCregLiveUpd(id->idGCref(), id->idReg2(), dst); - } - else - { - emitGCregDeadUpd(id->idReg2(), dst); - } - - goto SKIP_GC_UPDATE; - } - - break; - - case IF_LS_3F: // LS_3F .Q.........mmmmm ....ssnnnnnttttt Vt Rn Rm - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeVLSElemsize(elemsize); // ss - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - - dst += emitOutput_Instr(dst, code); - break; - - case IF_LS_3G: // LS_3G .Q.........mmmmm ...Sssnnnnnttttt Vt[] Rn Rm - elemsize = id->idOpSize(); - index = id->idSmallCns(); - code = emitInsCode(ins, fmt); - - code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vt(id->idReg1()); // ttttt - - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_1A: // DI_1A X.......shiiiiii iiiiiinnnnn..... Rn imm(i12,sh) - assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); - imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftImm12(id->idInsOpt()); // sh - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_1B: // DI_1B X........hwiiiii iiiiiiiiiiiddddd Rd imm(i16,hw) - imm = emitGetInsSC(id); - assert(isValidImmHWVal(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= ((code_t)imm << 5); // hwiiiii iiiiiiiiiii - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_1C: // DI_1C X........Nrrrrrr ssssssnnnnn..... Rn imm(N,r,s) - imm = emitGetInsSC(id); - assert(isValidImmNRS(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_1D: // DI_1D X........Nrrrrrr ssssss.....ddddd Rd imm(N,r,s) - imm = emitGetInsSC(id); - assert(isValidImmNRS(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_1E: // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 - case IF_LARGEADR: - assert(insOptsNone(id->idInsOpt())); - if (id->idIsReloc()) - { - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21 - : IMAGE_REL_ARM64_PAGEBASE_REL21); - } - else - { - // Local jmp/load case which does not need a relocation. - assert(id->idIsBound()); - dst = emitOutputLJ(ig, dst, id); - } - sz = sizeof(instrDescJmp); - break; - - case IF_DI_1F: // DI_1F X..........iiiii cccc..nnnnn.nzcv Rn imm5 nzcv cond - imm = emitGetInsSC(id); - assert(isValidImmCondFlagsImm5(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= ((code_t)cfi.imm5 << 16); // iiiii - code |= insEncodeFlags(cfi.flags); // nzcv - code |= insEncodeCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; - - case IF_DI_2A: // DI_2A X.......shiiiiii iiiiiinnnnnddddd Rd Rn imm(i12,sh) - assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); - imm = emitGetInsSC(id); - assert(isValidUimm<12>(imm)); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftImm12(id->idInsOpt()); // sh - code |= ((code_t)imm << 10); // iiiiiiiiiiii - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - - if (id->idIsReloc()) - { - assert(sz == sizeof(instrDesc)); - assert(id->idAddr()->iiaAddr != nullptr); - emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADD_LO12 - : IMAGE_REL_ARM64_PAGEOFFSET_12A); - } - break; - - case IF_DI_2B: // DI_2B X.........Xnnnnn ssssssnnnnnddddd Rd Rn imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // Reg2 also in mmmmm - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_2C: // DI_2C X........Nrrrrrr ssssssnnnnnddddd Rd Rn imm(N,r,s) - imm = emitGetInsSC(id); - assert(isValidImmNRS(imm, id->idOpSize())); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DI_2D: // DI_2D X........Nrrrrrr ssssssnnnnnddddd Rd Rn imr, imms (N,r,s) - if (ins == INS_asr || ins == INS_lsl || ins == INS_lsr) - { - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - - // Shift immediates are aliases of the SBFM/UBFM instructions - // that actually take 2 registers and 2 constants, - // Since we stored the shift immediate value - // we need to calculate the N,R and S values here. - - bitMaskImm bmi; - bmi.immNRS = 0; - - bmi.immN = (size == EA_8BYTE) ? 1 : 0; - bmi.immR = imm; - bmi.immS = (size == EA_8BYTE) ? 0x3f : 0x1f; - - // immR and immS are now set correctly for INS_asr and INS_lsr - // but for INS_lsl we have to adjust the values for immR and immS - // - if (ins == INS_lsl) - { - bmi.immR = -imm & bmi.immS; - bmi.immS = bmi.immS - imm; - } - - // setup imm with the proper 13 bit value N:R:S - // - imm = bmi.immNRS; - } - else - { - // The other instructions have already have encoded N,R and S values - imm = emitGetInsSC(id); - } - assert(isValidImmNRS(imm, id->idOpSize())); - - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 10); // Nrrrrrrssssss - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_1D: // DR_1D X............... cccc.......ddddd Rd cond - imm = emitGetInsSC(id); - assert(isValidImmCond(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeInvertedCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; - - case IF_DR_2A: // DR_2A X..........mmmmm ......nnnnn..... Rn Rm - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2B: // DR_2B X.......sh.mmmmm ssssssnnnnn..... Rn Rm {LSL,LSR,ASR,ROR} imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftType(id->idInsOpt()); // sh - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2C: // DR_2C X..........mmmmm ooosssnnnnn..... Rn Rm ext(Rm) LSL imm(0-4) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert((imm >= 0) && (imm <= 4)); // imm [0..4] - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeExtend(id->idInsOpt()); // ooo - code |= insEncodeExtendScale(imm); // sss - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2D: // DR_2D X..........nnnnn cccc..nnnnnddddd Rd Rn cond - imm = emitGetInsSC(id); - assert(isValidImmCond(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - code |= insEncodeInvertedCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; - - case IF_DR_2E: // DR_2E X..........mmmmm ...........ddddd Rd Rm - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2F: // DR_2F X.......sh.mmmmm ssssss.....ddddd Rd Rm {LSL,LSR,ASR} imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeShiftType(id->idInsOpt()); // sh - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2G: // DR_2G X............... .....xnnnnnddddd Rd Rn - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - if (ins == INS_rev) - { - if (size == EA_8BYTE) - { - code |= 0x00000400; // x - bit at location 10 - } - } - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2H: // DR_2H X........X...... ......nnnnnddddd Rd Rn - code = emitInsCode(ins, fmt); - code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_2I: // DR_2I X..........mmmmm cccc..nnnnn.nzcv Rn Rm nzcv cond - imm = emitGetInsSC(id); - assert(isValidImmCondFlags(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - code |= insEncodeReg_Rm(id->idReg2()); // mmmmm - code |= insEncodeFlags(cfi.flags); // nzcv - code |= insEncodeCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; - - case IF_DR_3A: // DR_3A X..........mmmmm ......nnnnnmmmmm Rd Rn Rm - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - if (id->idIsLclVar()) - { - code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm - } - else - { - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - } - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_3B: // DR_3B X.......sh.mmmmm ssssssnnnnnddddd Rd Rn Rm {LSL,LSR,ASR} imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeShiftType(id->idInsOpt()); // sh - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_3C: // DR_3C X..........mmmmm ooosssnnnnnddddd Rd Rn Rm ext(Rm) LSL imm(0-4) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert((imm >= 0) && (imm <= 4)); // imm [0..4] - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeExtend(id->idInsOpt()); // ooo - code |= insEncodeExtendScale(imm); // sss - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_3D: // DR_3D X..........mmmmm cccc..nnnnnddddd Rd Rn Rm cond - imm = emitGetInsSC(id); - assert(isValidImmCond(imm)); - { - condFlagsImm cfi; - cfi.immCFVal = (unsigned)imm; - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeCond(cfi.cond); // cccc - dst += emitOutput_Instr(dst, code); - } - break; - - case IF_DR_3E: // DR_3E X........X.mmmmm ssssssnnnnnddddd Rd Rn Rm imm(0-63) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - assert(isValidImmShift(imm, id->idOpSize())); - code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss - dst += emitOutput_Instr(dst, code); - break; - - case IF_DR_4A: // DR_4A X..........mmmmm .aaaaannnnnmmmmm Rd Rn Rm Ra - code = emitInsCode(ins, fmt); - code |= insEncodeDatasize(id->idOpSize()); // X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeReg_Ra(id->idReg4()); // aaaaa - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_1A: // DV_1A .........X.iiiii iii........ddddd Vd imm8 (fmov - immediate scalar) - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= ((code_t)imm << 13); // iiiii iii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_1B: // DV_1B .QX..........iii cmod..iiiiiddddd Vd imm8 (immediate vector) - imm = emitGetInsSC(id) & 0x0ff; - immShift = (emitGetInsSC(id) & 0x700) >> 8; - elemsize = optGetElemsize(id->idInsOpt()); - cmode = 0; - switch (elemsize) - { // cmode - case EA_1BYTE: - cmode = 0xE; // 1110 - break; - case EA_2BYTE: - cmode = 0x8; - cmode |= (immShift << 1); // 10x0 - break; - case EA_4BYTE: - if (immShift < 4) - { - cmode = 0x0; - cmode |= (immShift << 1); // 0xx0 - } - else // MSL - { - cmode = 0xC; - if (immShift & 2) - cmode |= 1; // 110x - } - break; - case EA_8BYTE: - cmode = 0xE; // 1110 - break; - default: - unreached(); - break; - } - - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - if ((ins == INS_fmov) || (ins == INS_movi)) - { - if (elemsize == EA_8BYTE) - { - code |= 0x20000000; // X - } - } - if (ins != INS_fmov) - { - assert((cmode >= 0) && (cmode <= 0xF)); - code |= (cmode << 12); // cmod - } - code |= (((code_t)imm >> 5) << 16); // iii - code |= (((code_t)imm & 0x1f) << 5); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_1C: // DV_1C .........X...... ......nnnnn..... Vn #0.0 (fcmp - with zero) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vn(id->idReg1()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2A: // DV_2A .Q.......X...... ......nnnnnddddd Vd Vn (fabs, fcvt - vector) - case IF_DV_2R: // DV_2R .Q.......X...... ......nnnnnddddd Sd Vn (fmaxnmv, fmaxv, fminnmv, fminv) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - if ((ins == INS_fcvtl) || (ins == INS_fcvtl2) || (ins == INS_fcvtn) || (ins == INS_fcvtn2)) - { - // fcvtl{2} and fcvtn{2} encode the element size as - // esize = 16 << UInt(sz) - if (elemsize == EA_4BYTE) - { - code |= 0x00400000; // X - } - else - { - assert(elemsize == EA_2BYTE); - } - } - else - { - code |= insEncodeFloatElemsize(elemsize); // X - } - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2B: // DV_2B .Q.........iiiii ......nnnnnddddd Rd Vn[] (umov/smov - to general) - elemsize = id->idOpSize(); - index = emitGetInsSC(id); - datasize = (elemsize == EA_8BYTE) ? EA_16BYTE : EA_8BYTE; - if (ins == INS_smov) - { - datasize = EA_16BYTE; - } - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(datasize); // Q - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2C: // DV_2C .Q.........iiiii ......nnnnnddddd Vd Rn (dup/ins - vector from general) - if (ins == INS_dup) - { - datasize = id->idOpSize(); - elemsize = optGetElemsize(id->idInsOpt()); - index = 0; - } - else // INS_ins - { - datasize = EA_16BYTE; - elemsize = id->idOpSize(); - index = emitGetInsSC(id); - } - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(datasize); // Q - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2D: // DV_2D .Q.........iiiii ......nnnnnddddd Vd Vn[] (dup - vector) - index = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2E: // DV_2E ...........iiiii ......nnnnnddddd Vd Vn[] (dup - scalar) - index = emitGetInsSC(id); - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2F: // DV_2F ...........iiiii .jjjj.nnnnnddddd Vd[] Vn[] (ins - element) - elemsize = id->idOpSize(); - imm = emitGetInsSC(id); - index = (imm >> 4) & 0xf; - index2 = imm & 0xf; - code = emitInsCode(ins, fmt); - code |= insEncodeVectorIndex(elemsize, index); // iiiii - code |= insEncodeVectorIndex2(elemsize, index2); // jjjj - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2G: // DV_2G .........X...... ......nnnnnddddd Vd Vn (fmov, fcvtXX - register) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2H: // DV_2H X........X...... ......nnnnnddddd Rd Vn (fmov - to general) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2I: // DV_2I X........X...... ......nnnnnddddd Vd Rn (fmov - from general) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2J: // DV_2J ........SS.....D D.....nnnnnddddd Vd Vn (fcvt) - code = emitInsCode(ins, fmt); - code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // SS DD - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2K: // DV_2K .........X.mmmmm ......nnnnn..... Vn Vm (fcmp) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vn(id->idReg1()); // nnnnn - code |= insEncodeReg_Vm(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2L: // DV_2L ........XX...... ......nnnnnddddd Vd Vn (abs, neg - scalar) - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2M: // DV_2M .Q......XX...... ......nnnnnddddd Vd Vn (abs, neg - vector) - case IF_DV_2T: // DV_2T .Q......XX...... ......nnnnnddddd Sd Vn (addv, saddlv, smaxv, sminv, uaddlv, - // umaxv, uminv) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2N: // DV_2N .........iiiiiii ......nnnnnddddd Vd Vn imm (shift - scalar) - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2O: // DV_2O .Q.......iiiiiii ......nnnnnddddd Vd Vn imm (shift - vector) - imm = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2P: // DV_2P ............... ......nnnnnddddd Vd Vn (aes*, sha1su1) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2Q: // DV_2Q .........X...... ......nnnnnddddd Vd Vn (faddp, fmaxnmp, fmaxp, fminnmp, - // fminp - scalar) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2S: // DV_2S ........XX...... ......nnnnnddddd Sd Vn (addp - scalar) - elemsize = optGetElemsize(id->idInsOpt()); - code = emitInsCode(ins, fmt); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_2U: // DV_2U ................ ......nnnnnddddd Sd Sn (sha1h) - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3A: // DV_3A .Q......XX.mmmmm ......nnnnnddddd Vd Vn Vm (vector) - code = emitInsCode(ins, fmt); - elemsize = optGetElemsize(id->idInsOpt()); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3AI: // DV_3AI .Q......XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3B: // DV_3B .Q.......X.mmmmm ......nnnnnddddd Vd Vn Vm (vector) - code = emitInsCode(ins, fmt); - elemsize = optGetElemsize(id->idInsOpt()); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3BI: // DV_3BI .Q.......XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector by element) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = optGetElemsize(id->idInsOpt()); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeFloatIndex(elemsize, imm); // L H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector) - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3D: // DV_3D .........X.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) - code = emitInsCode(ins, fmt); - code |= insEncodeFloatElemsize(id->idOpSize()); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3DI: // DV_3DI .........XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeFloatIndex(elemsize, imm); // L H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3E: // DV_3E ........XX.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) - code = emitInsCode(ins, fmt); - elemsize = id->idOpSize(); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3EI: // DV_3EI ........XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) - code = emitInsCode(ins, fmt); - imm = emitGetInsSC(id); - elemsize = id->idOpSize(); - assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); - code |= insEncodeElemsize(elemsize); // XX - code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3F: // DV_3F ...........mmmmm ......nnnnnddddd Vd Vn Vm (vector) - source dest regs overlap - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_3G: // DV_3G .Q.........mmmmm .iiii.nnnnnddddd Vd Vn Vm imm (vector) - imm = emitGetInsSC(id); - code = emitInsCode(ins, fmt); - code |= insEncodeVectorsize(id->idOpSize()); // Q - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - code |= ((code_t)imm << 11); // iiii - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - dst += emitOutput_Instr(dst, code); - break; - - case IF_DV_4A: // DV_4A .........X.mmmmm .aaaaannnnnddddd Vd Va Vn Vm (scalar) - code = emitInsCode(ins, fmt); - elemsize = id->idOpSize(); - code |= insEncodeFloatElemsize(elemsize); // X - code |= insEncodeReg_Vd(id->idReg1()); // ddddd - code |= insEncodeReg_Vn(id->idReg2()); // nnnnn - code |= insEncodeReg_Vm(id->idReg3()); // mmmmm - code |= insEncodeReg_Va(id->idReg4()); // aaaaa - dst += emitOutput_Instr(dst, code); - break; - - case IF_SN_0A: // SN_0A ................ ................ - { - bool skipIns = false; -#if FEATURE_LOOP_ALIGN - if (id->idIns() == INS_align) - { - // IG can be marked as not needing alignment after emitting align instruction. - // Alternatively, there are fewer align instructions needed than emitted. - // If that is the case, skip outputting alignment. - if (!ig->endsWithAlignInstr() || id->idIsEmptyAlign()) - { - skipIns = true; - } - -#ifdef DEBUG - if (!ig->endsWithAlignInstr()) - { - // Validate if the state is correctly updated - assert(id->idIsEmptyAlign()); - } -#endif - sz = sizeof(instrDescAlign); - ins = INS_nop; - -#ifdef DEBUG - // Under STRESS_EMITTER, if this is the 'align' before the 'jmp' instruction, - // then add "bkpt" instruction. - instrDescAlign* alignInstr = (instrDescAlign*)id; - - if (emitComp->compStressCompile(Compiler::STRESS_EMITTER, 50) && alignInstr->isPlacedAfterJmp && - !skipIns) - { - // There is no good way to squeeze in "bkpt" as well as display it - // in the disassembly because there is no corresponding instrDesc for - // it. As such, leave it as is, the "0xD43E0000" bytecode will be seen - // next to the nop instruction in disasm. - // e.g. D43E0000 align [4 bytes for IG07] - ins = INS_BREAKPOINT; - fmt = IF_SI_0A; - } -#endif - } -#endif // FEATURE_LOOP_ALIGN - - if (!skipIns) - { - code = emitInsCode(ins, fmt); - dst += emitOutput_Instr(dst, code); - } - - break; - } - - case IF_SI_0A: // SI_0A ...........iiiii iiiiiiiiiii..... imm16 - imm = emitGetInsSC(id); - assert(isValidUimm<16>(imm)); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 5); // iiiii iiiiiiiiiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SI_0B: // SI_0B ................ ....bbbb........ imm4 - barrier - imm = emitGetInsSC(id); - assert((imm >= 0) && (imm <= 15)); - code = emitInsCode(ins, fmt); - code |= ((code_t)imm << 8); // bbbb - dst += emitOutput_Instr(dst, code); - break; - - case IF_SR_1A: // SR_1A ................ ...........ttttt Rt (dc zva, mrs) - assert(insOptsNone(id->idInsOpt())); - code = emitInsCode(ins, fmt); - code |= insEncodeReg_Rt(id->idReg1()); // ttttt - dst += emitOutput_Instr(dst, code); - break; - - default: - dst = emitOutput_InstrSve(dst, id); - break; - } - - // Determine if any registers now hold GC refs, or whether a register that was overwritten held a GC ref. - // We assume here that "id->idGCref()" is not GC_NONE only if the instruction described by "id" writes a - // GC ref to register "id->idReg1()". (It may, apparently, also not be GC_NONE in other cases, such as - // for stores, but we ignore those cases here.) - if (emitInsMayWriteToGCReg(id)) // True if "id->idIns()" writes to a register than can hold GC ref. - { - // We assume that "idReg1" is the primary destination register for all instructions - assert(!emitInsDestIsOp2(ins)); - if (id->idGCref() != GCT_NONE) - { - emitGCregLiveUpd(id->idGCref(), id->idReg1(), dst); - } - else - { - emitGCregDeadUpd(id->idReg1(), dst); - } - - if (emitInsMayWriteMultipleRegs(id)) - { - // INS_ldp etc... - // "idReg2" is the secondary destination register - if (id->idGCrefReg2() != GCT_NONE) - { - emitGCregLiveUpd(id->idGCrefReg2(), id->idReg2(), dst); - } - else - { - emitGCregDeadUpd(id->idReg2(), dst); - } - } - } - -SKIP_GC_UPDATE: - // Now we determine if the instruction has written to a (local variable) stack location, and either written a GC - // ref or overwritten one. - if (emitInsWritesToLclVarStackLoc(id) || emitInsWritesToLclVarStackLocPair(id)) - { - int varNum = id->idAddr()->iiaLclVar.lvaVarNum(); - unsigned ofs = AlignDown(id->idAddr()->iiaLclVar.lvaOffset(), TARGET_POINTER_SIZE); - bool FPbased; - int adr = emitComp->lvaFrameAddress(varNum, &FPbased); - if (id->idGCref() != GCT_NONE) + // Is the target a vector register? + if (isVectorRegister(reg)) { - emitGCvarLiveUpd(adr + ofs, varNum, id->idGCref(), dst DEBUG_ARG(varNum)); + code |= insEncodeDatasizeVLS(code, opSize); // XX V + code |= insEncodeReg_Vt(reg); // ttttt } else { - // If the type of the local is a gc ref type, update the liveness. - var_types vt; - if (varNum >= 0) - { - // "Regular" (non-spill-temp) local. - vt = var_types(emitComp->lvaTable[varNum].lvType); - } - else - { - TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum); - vt = tmpDsc->tdTempType(); - } - if (vt == TYP_REF || vt == TYP_BYREF) - { - emitGCvarDeadUpd(adr + ofs, dst DEBUG_ARG(varNum)); - } - } - if (emitInsWritesToLclVarStackLocPair(id)) - { - int varNum2 = varNum; - int adr2 = adr; - unsigned ofs2 = ofs; - unsigned ofs2Dist; - - if (id->idIsLclVarPair()) - { - bool FPbased2; - - emitLclVarAddr* lclVarAddr2 = emitGetLclVarPairLclVar2(id); - varNum2 = lclVarAddr2->lvaVarNum(); - ofs2 = lclVarAddr2->lvaOffset(); - - // If there are 2 GC vars in this instrDesc, get the 2nd variable - // that should be tracked. - adr2 = emitComp->lvaFrameAddress(varNum2, &FPbased2); - ofs2Dist = EA_SIZE_IN_BYTES(size); -#ifdef DEBUG - assert(FPbased == FPbased2); - if (FPbased) - { - assert(id->idReg3() == REG_FP); - } - else - { - assert(id->idReg3() == REG_SP); - } - assert(varNum2 != -1); -#endif // DEBUG - } - else - { - ofs2Dist = TARGET_POINTER_SIZE; - ofs2 += ofs2Dist; - } - - ofs2 = AlignDown(ofs2, ofs2Dist); - - if (id->idGCrefReg2() != GCT_NONE) - { -#ifdef DEBUG - if (id->idGCref() != GCT_NONE) - { - // If 1st register was a gc-var, then make sure the offset - // are correctly set for the 2nd register that is holding - // another gc-var. - assert((adr + ofs + ofs2Dist) == (adr2 + ofs2)); - } -#endif - emitGCvarLiveUpd(adr2 + ofs2, varNum2, id->idGCrefReg2(), dst DEBUG_ARG(varNum2)); - } - else + assert(isGeneralRegister(reg)); + // insEncodeDatasizeLS is not quite right for this case. + // So just specialize it. + if ((ins == INS_ldr) && (opSize == EA_8BYTE)) { - // If the type of the local is a gc ref type, update the liveness. - var_types vt; - if (varNum2 >= 0) - { - // "Regular" (non-spill-temp) local. - vt = var_types(emitComp->lvaTable[varNum2].lvType); - } - else - { - TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum2); - vt = tmpDsc->tdTempType(); - } - if (vt == TYP_REF || vt == TYP_BYREF) - { - emitGCvarDeadUpd(adr2 + ofs2, dst DEBUG_ARG(varNum2)); - } + // set the operation size in bit 30 + code |= 0x40000000; } - } - } - -#ifdef DEBUG - /* Make sure we set the instruction descriptor size correctly */ - size_t expected = emitSizeOfInsDsc(id); - assert(sz == expected); + code |= insEncodeReg_Rt(reg); // ttttt + } - if (emitComp->opts.disAsm || emitComp->verbose) - { - emitDispIns(id, false, dspOffs, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + distVal &= 0x7FFFFLL; // 19 bits + code |= distVal << 5; } - - if (emitComp->compDebugBreak) + else if (fmt == IF_LS_2B) { - // For example, set JitBreakEmitOutputInstr=a6 will break when this method is called for - // emitting instruction a6, (i.e. IN00a6 in jitdump). - if ((unsigned)JitConfig.JitBreakEmitOutputInstr() == id->idDebugOnlyInfo()->idNum) + // ldr Rt,[Xn+pimm12] LS_2B 1X11100101iiiiii iiiiiinnnnnttttt B940 0000 imm(0-4095<<{2,3}) + // INS_ldr or INS_ldrsw (PC-Relative) + noway_assert(isValidUimm<12>(imm)); + assert(isGeneralRegister(reg)); + + if (opSize == EA_8BYTE) { - assert(!"JitBreakEmitOutputInstr reached"); + // insEncodeDatasizeLS is not quite right for this case. + // So just specialize it. + if (ins == INS_ldr) + { + // set the operation size in bit 30 + code |= 0x40000000; + } + // Low 3 bits should be 0 -- 8 byte JIT data should be aligned on 8 byte. + assert((imm & 7) == 0); + imm >>= 3; + } + else + { + assert(opSize == EA_4BYTE); + // Low 2 bits should be 0 -- 4 byte aligned data. + assert((imm & 3) == 0); + imm >>= 2; } - } - // Output any delta in GC info. - if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) - { - emitDispGCInfoDelta(); + code |= insEncodeReg_Rt(reg); // ttttt + code |= insEncodeReg_Rn(reg); // nnnnn + code |= imm << 10; } -#else - if (emitComp->opts.disAsm) + else { - size_t expected = emitSizeOfInsDsc(id); - assert(sz == expected); - emitDispIns(id, false, 0, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + assert(!"Unknown fmt for emitOutputShortConstant"); } -#endif - - /* All instructions are expected to generate code */ - - assert(*dp != dst || id->idIsEmptyAlign()); - *dp = dst; + dst += emitOutput_Instr(dst, code); - return sz; + return dst; } /***************************************************************************** * - * Append the machine code corresponding to the given SVE instruction descriptor. + * Output instructions to load a constant into a vector register. */ -BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) +BYTE* emitter::emitOutputVectorConstant( + BYTE* dst, ssize_t imm, regNumber dstReg, regNumber addrReg, emitAttr opSize, emitAttr elemSize) { - code_t code = 0; - instruction ins = id->idIns(); - insFormat fmt = id->idInsFmt(); - emitAttr size = id->idOpSize(); - - ssize_t imm; - - switch (fmt) - { - // Scalable. - case IF_SVE_AA_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise logical operations (predicated) - case IF_SVE_AB_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_AC_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer divide vectors (predicated) - case IF_SVE_AD_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer min/max/difference (predicated) - case IF_SVE_AE_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer multiply vectors (predicated) - case IF_SVE_AF_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (predicated) - case IF_SVE_AG_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (quadwords) - case IF_SVE_AI_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (predicated) - case IF_SVE_AJ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (quadwords) - case IF_SVE_AK_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (predicated) - case IF_SVE_AL_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (quadwords) - case IF_SVE_AN_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by vector (predicated) - case IF_SVE_AO_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by wide elements (predicated) - case IF_SVE_AP_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise unary operations (predicated) - case IF_SVE_AQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer unary operations (predicated) - case IF_SVE_CL_3A: // ........xx...... ...gggnnnnnddddd -- SVE compress active elements - case IF_SVE_CM_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally broadcast element to vector - case IF_SVE_CN_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to SIMD&FP scalar - case IF_SVE_CP_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy SIMD&FP scalar register to vector - // (predicated) - case IF_SVE_CR_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to SIMD&FP scalar register - case IF_SVE_CU_3A: // ........xx...... ...gggnnnnnddddd -- SVE reverse within elements - case IF_SVE_EP_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer halving add/subtract (predicated) - case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long - case IF_SVE_ER_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer pairwise arithmetic - case IF_SVE_ES_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer unary operations (predicated) - case IF_SVE_ET_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating add/subtract - case IF_SVE_EU_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating/rounding bitwise shift left - // (predicated) - case IF_SVE_GR_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 floating-point pairwise operations - case IF_SVE_GS_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction (quadwords) - case IF_SVE_HE_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction - case IF_SVE_HJ_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point serial reduction (predicated) - case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - case IF_SVE_HQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point round to integral value - case IF_SVE_HR_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point unary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm or nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) - case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); - break; - - // Scalable with Merge or Zero predicate - case IF_SVE_AH_3A: // ........xx.....M ...gggnnnnnddddd -- SVE constructive prefix (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // nnnnn - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // ddddd - code |= insEncodePredQualifier_16(id->idPredicateReg2Merge()); // M - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - // Scalable with shift immediate - case IF_SVE_AM_2A: // ........xx...... ...gggxxiiiddddd -- SVE bitwise shift by immediate (predicated) - { - bool isRightShift = emitInsIsVectorRightShift(ins); - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= - insEncodeSveShift_23_to_22_9_to_0(optGetSveElemsize(id->idInsOpt()), isRightShift, imm); // xx, xxiii - dst += emitOutput_Instr(dst, code); - } - break; - - // Scalable, 4 regs. Reg4 in mmmmm. - case IF_SVE_AR_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE integer multiply-accumulate writing addend - // (predicated) - case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) - case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - // Scalable, 4 regs. Reg4 in aaaaa. - case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand - // (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeReg_V_9_to_5(id->idReg4()); // aaaaa - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - // Scalable, 3 regs, no predicates - case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high - // (unpredicated) - case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) - case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient - case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) - case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads - case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) - case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long - case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high - case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long - case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long - case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp - case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) - case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long - case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide - case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved - case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute - case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long - case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate - case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long - case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part - case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) - case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - // Scalable, 3 regs, no predicates. General purpose source registers - case IF_SVE_BA_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE index generation (register start, register - // increment) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeReg_Rm(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh - code |= insEncodeUimm<22, 22>(id->idInsOpt() == INS_OPTS_SCALABLE_D ? 1 : 0); - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh - dst += emitOutput_Instr(dst, code); - break; - - // Immediate and pattern to general purpose. - case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count - case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - code |= insEncodeSveElemsize_sz_20(id->idOpSize()); // X - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) - case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn/mmmmm - code |= insEncodeUimm<12, 10>(imm & 0b111); // iii - code |= insEncodeUimm<20, 16>(imm >> 3); // iiiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count - case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeUimm4From1_19_to_16(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) - case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= (imm << 5); - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // add addrReg, addrReg, page offs -- compute address = page addr + page offs. + code_t code = emitInsCode(INS_add, IF_DI_2A); // DI_2A X0010001shiiiiii iiiiiinnnnnddddd 1100 0000 imm(i12, sh) + code |= insEncodeDatasize(EA_8BYTE); // X - use EA_8BYTE, as we are calculating 64-bit address + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(addrReg); // ddddd + code |= insEncodeReg_Rn(addrReg); // nnnnn + dst += emitOutput_Instr(dst, code); - case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= (id->idHasShift() ? 0x2000 : 0); // h - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // ld1 dstReg, addrReg -- load constant at address in addrReg into dstReg. + code = emitInsCode(INS_ld1, IF_LS_2D); // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn + code |= insEncodeVectorsize(opSize); // Q + code |= insEncodeVLSElemsize(elemSize); // ss + code |= insEncodeReg_Rn(addrReg); // nnnnn + code |= insEncodeReg_Vt(dstReg); // ttttt + dst += emitOutput_Instr(dst, code); - case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) - // In emitIns, we set this format's instruction to MOV, as that is the preferred disassembly. - // However, passing (MOV, IF_SVE_BV_2B) to emitInsCodeSve will assert with "encoding_found", - // as FMOV is the only instruction associated with this encoding format. - // Thus, always pass FMOV here, and use MOV elsewhere for simplicity. - code = emitInsCodeSve(INS_sve_fmov, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + return dst; +} - case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveBroadcastIndex(optGetSveElemsize(id->idInsOpt()), imm); - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** + * + * Output a call instruction. + */ - case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +unsigned emitter::emitOutputCall(insGroup* ig, BYTE* dst, instrDesc* id, code_t code) +{ + const unsigned char callInstrSize = sizeof(code_t); // 4 bytes + regMaskTP gcrefRegs; + regMaskTP byrefRegs; - case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii - dst += emitOutput_Instr(dst, code); - break; + VARSET_TP GCvars(VarSetOps::UninitVal()); - case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i - dst += emitOutput_Instr(dst, code); - break; + // Is this a "fat" call descriptor? + if (id->idIsLargeCall()) + { + instrDescCGCA* idCall = (instrDescCGCA*)id; + gcrefRegs = idCall->idcGcrefRegs; + byrefRegs = idCall->idcByrefRegs; + VarSetOps::Assign(emitComp, GCvars, idCall->idcGCvars); + } + else + { + assert(!id->idIsLargeDsp()); + assert(!id->idIsLargeCns()); - case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; + gcrefRegs = emitDecodeCallGCregs(id); + byrefRegs = 0; + VarSetOps::AssignNoCopy(emitComp, GCvars, VarSetOps::MakeEmpty(emitComp)); + } - case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + /* We update the GC info before the call as the variables cannot be + used by the call. Killing variables before the call helps with + boundary conditions if the call is CORINFO_HELP_THROW - see bug 50029. + If we ever track aliased variables (which could be used by the + call), we would have to keep them alive past the call. */ - case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii - dst += emitOutput_Instr(dst, code); - break; + emitUpdateLiveGCvars(GCvars, dst); - case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG + // Output any delta in GC variable info, corresponding to the before-call GC var updates done above. + if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) + { + emitDispGCVarDelta(); + } +#endif // DEBUG - case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; + // Now output the call instruction and update the 'dst' pointer + // + unsigned outputInstrSize = emitOutput_Instr(dst, code); + dst += outputInstrSize; - case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // All call instructions are 4-byte in size on ARM64 + // + assert(outputInstrSize == callInstrSize); - case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // If the method returns a GC ref, mark INTRET (R0) appropriately. + if (id->idGCref() == GCT_GCREF) + { + gcrefRegs |= RBM_INTRET; + } + else if (id->idGCref() == GCT_BYREF) + { + byrefRegs |= RBM_INTRET; + } - case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeReg_P_19_to_16(id->idReg3()); // MMMM - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // If is a multi-register return method is called, mark INTRET_1 (X1) appropriately + if (id->idIsLargeCall()) + { + instrDescCGCA* idCall = (instrDescCGCA*)id; + if (idCall->idSecondGCref() == GCT_GCREF) + { + gcrefRegs |= RBM_INTRET_1; + } + else if (idCall->idSecondGCref() == GCT_BYREF) + { + byrefRegs |= RBM_INTRET_1; + } + } - case IF_SVE_CJ_2A: // ........xx...... .......nnnn.dddd -- SVE reverse predicate elements - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // If the GC register set has changed, report the new set. + if (gcrefRegs != emitThisGCrefRegs) + { + emitUpdateLiveGCregs(GCT_GCREF, gcrefRegs, dst); + } + // If the Byref register set has changed, report the new set. + if (byrefRegs != emitThisByrefRegs) + { + emitUpdateLiveGCregs(GCT_BYREF, byrefRegs, dst); + } - case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + // Some helper calls may be marked as not requiring GC info to be recorded. + if ((!id->idIsNoGC())) + { + // On ARM64, as on AMD64, we don't change the stack pointer to push/pop args. + // So we're not really doing a "stack pop" here (note that "args" is 0), but we use this mechanism + // to record the call for GC info purposes. (It might be best to use an alternate call, + // and protect "emitStackPop" under the EMIT_TRACK_STACK_DEPTH preprocessor variable.) + emitStackPop(dst, /*isCall*/ true, callInstrSize, /*args*/ 0); - case IF_SVE_GQ_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision odd elements - code = emitInsCodeSve(ins, fmt); + // Do we need to record a call location for GC purposes? + // + if (!emitFullGCinfo) + { + emitRecordGCcall(dst, callInstrSize); + } + } + return callInstrSize; +} - if (ins == INS_sve_fcvtnt && id->idInsOpt() == INS_OPTS_D_TO_S) - { - code |= (1 << 22 | 1 << 17); - } - else if (ins == INS_sve_fcvtlt && id->idInsOpt() == INS_OPTS_S_TO_D) - { - code |= (1 << 22 | 1 << 17); - } +/***************************************************************************** + * + * Emit a 32-bit Arm64 instruction + */ - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +unsigned emitter::emitOutput_Instr(BYTE* dst, code_t code) +{ + assert(sizeof(code_t) == 4); + BYTE* dstRW = dst + writeableOffset; + *((code_t*)dstRW) = code; - // Scalable to general register. - case IF_SVE_CO_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to general register - case IF_SVE_CS_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_Rd(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + return sizeof(code_t); +} - // Scalable from general register. - case IF_SVE_CQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy general register to vector (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_Rn(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; +/***************************************************************************** +* + * Append the machine code corresponding to the given instruction descriptor + * to the code block at '*dp'; the base of the code block is 'bp', and 'ig' + * is the instruction group that contains the instruction. Updates '*dp' to + * point past the generated code, and returns the size of the instruction + * descriptor in bytes. + */ - case IF_SVE_CT_3A: // ................ ...gggnnnnnddddd -- SVE reverse doublewords - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; +size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp) +{ + BYTE* dst = *dp; + BYTE* odst = dst; + code_t code = 0; + size_t sz = emitGetInstrDescSize(id); // TODO-ARM64-Cleanup: on ARM, this is set in each case. why? + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); + emitAttr size = id->idOpSize(); - case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) - case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // VVV - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn/mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG +#if DUMP_GC_TABLES + bool dspOffs = emitComp->opts.dspGCtbls; +#else + bool dspOffs = !emitComp->opts.disDiffable; +#endif +#endif // DEBUG - case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) - { - regNumber reg4 = (ins == INS_sve_mov ? id->idReg1() : id->idReg4()); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_13_to_10(id->idReg2()); // VVVV - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(reg4); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } + assert(REG_NA == (int)REG_NA); - case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors - case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors - case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match - case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeReg_V_20_to_16(id->idReg4()); // nnnnn - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + /* What instruction format have we got? */ - case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm<20, 16>(imm); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + switch (fmt) + { + ssize_t imm; + ssize_t index; + ssize_t index2; + unsigned cmode; + unsigned immShift; + emitAttr elemsize; + emitAttr datasize; - case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm<20, 14>(imm); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_BI_0A: // BI_0A ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 + case IF_BI_0B: // BI_0B ......iiiiiiiiii iiiiiiiiiii..... simm19:00 + case IF_LARGEJMP: + assert(id->idGCref() == GCT_NONE); + assert(id->idIsBound()); + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) - case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments - case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long - case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate - case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) - case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) - case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) - case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product - case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product - case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations - case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long - case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long - case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp - case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product - case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long - case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate - case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - dst += emitOutput_Instr(dst, code); + case IF_BI_0C: // BI_0C ......iiiiiiiiii iiiiiiiiiiiiiiii simm26:00 + code = emitInsCode(ins, fmt); + sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); + dst += emitOutputCall(ig, dst, id, code); + // Always call RecordRelocation so that we wire in a JumpStub when we don't reach + emitRecordRelocation(odst, id->idAddr()->iiaAddr, IMAGE_REL_ARM64_BRANCH26); break; - case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeReg_V_9_to_5(id->idReg3()); // kkkkk - dst += emitOutput_Instr(dst, code); - break; + case IF_BI_1A: // BI_1A ......iiiiiiiiii iiiiiiiiiiittttt Rt simm19:00 + assert(insOptsNone(id->idInsOpt())); + assert(id->idIsBound()); - case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm - imm = insGetImmDiff(emitGetInsSC(id), id->idInsOpt()); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<20, 16>(imm & 0b11111); // xxiii - code |= insEncodeUimm<22, 22>(imm >> 5); // x - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx xx - dst += emitOutput_Instr(dst, code); + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate - // increment) - { - ssize_t imm1; - ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<9, 5>(imm1); // iiiii - code |= insEncodeSimm<20, 16>(imm2); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } + case IF_BI_1B: // BI_1B B.......bbbbbiii iiiiiiiiiiittttt Rt imm6, simm14:00 + assert(insOptsNone(id->idInsOpt())); + assert(id->idIsBound()); - case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register - // increment) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<9, 5>(emitGetInsSC(id)); // iiiii - code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate - // increment) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeSimm<20, 16>(emitGetInsSC(id)); // iiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + case IF_BR_1A: // BR_1A ................ ......nnnnn..... Rn + assert(insOptsNone(id->idInsOpt())); + assert((ins == INS_ret) || (ins == INS_br)); + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn - case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii - code |= insEncodeReg_R_20_to_16(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii - dst += emitOutput_Instr(dst, code); - break; + case IF_BR_1B: // BR_1B ................ ......nnnnn..... Rn + assert(insOptsNone(id->idInsOpt())); + assert((ins == INS_br_tail) || (ins == INS_blr)); + code = emitInsCode(ins, fmt); - case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg3()); // aaaaa - code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm - dst += emitOutput_Instr(dst, code); + if (emitComp->IsTargetAbi(CORINFO_NATIVEAOT_ABI) && id->idIsTlsGD()) + { + emitRecordRelocation(odst, (CORINFO_METHOD_HANDLE)id->idAddr()->iiaAddr, + IMAGE_REL_AARCH64_TLSDESC_CALL); + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + } + else + { + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn + } + dst += emitOutputCall(ig, dst, id, code); + sz = id->idIsLargeCall() ? sizeof(instrDescCGCA) : sizeof(instrDesc); break; - case IF_SVE_EG_3A: // ...........iimmm ......nnnnnddddd -- SVE two-way dot product (indexed) - case IF_SVE_EY_3A: // ...........iimmm ......nnnnnddddd -- SVE integer dot product (indexed) - case IF_SVE_EZ_3A: // ...........iimmm ......nnnnnddddd -- SVE mixed sign dot product (indexed) - case IF_SVE_FD_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) - case IF_SVE_FF_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) - case IF_SVE_FI_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) - case IF_SVE_GU_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_GY_3B: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) - case IF_SVE_GY_3B_D: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) - case IF_SVE_FK_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(emitGetInsSC(id)); // ii - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_1A: // LS_1A XX...V..iiiiiiii iiiiiiiiiiittttt Rt PC imm(1MB) + case IF_LARGELDC: + assert(insOptsNone(id->idInsOpt())); + assert(id->idIsBound()); - case IF_SVE_FD_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) - case IF_SVE_FF_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) - case IF_SVE_FI_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) - case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_FK_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(imm & 0b11); // ii - code |= insEncodeUimm<22, 22>(imm >> 2); // i - dst += emitOutput_Instr(dst, code); + dst = emitOutputLJ(ig, dst, id); + sz = sizeof(instrDescJmp); break; - case IF_SVE_FE_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) - case IF_SVE_FG_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) - case IF_SVE_FH_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) - case IF_SVE_FJ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) - case IF_SVE_GY_3A: // ...........iimmm ....i.nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) - case IF_SVE_GZ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE floating-point multiply-add long (indexed) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 11>(imm & 1); // i - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(imm >> 1); // ii + case IF_LS_2A: // LS_2A .X.......X...... ......nnnnnttttt Rt Rn + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); + if (id->idIsTlsGD()) + { + emitRecordRelocation(odst, (void*)emitGetInsSC(id), IMAGE_REL_AARCH64_TLSDESC_LD64_LO12); + } break; - case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) - case IF_SVE_FG_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) - case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) - case IF_SVE_FJ_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 11>(imm & 1); // i - code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm - code |= insEncodeUimm<20, 19>(imm & 0b10); // i + case IF_LS_2B: // LS_2B .X.......Xiiiiii iiiiiinnnnnttttt Rt Rn imm(0-4095) + assert(insOptsNone(id->idInsOpt())); + imm = emitGetInsSC(id); + assert(isValidUimm<12>(imm)); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EY_3B: // ...........immmm ......nnnnnddddd -- SVE integer dot product (indexed) - case IF_SVE_FD_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply (indexed) - case IF_SVE_FF_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) - case IF_SVE_FI_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) - case IF_SVE_GU_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) - case IF_SVE_GX_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply (indexed) - case IF_SVE_FK_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm - - // index is encoded at bit location 20; - // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 - code |= insEncodeUimm<20, 19>(emitGetInsSC(id) << 1); // i + case IF_LS_2C: // LS_2C .X.......X.iiiii iiiiPPnnnnnttttt Rt Rn imm(-256..+255) no/pre/post inc + assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); + imm = emitGetInsSC(id); + assert((imm >= -256) && (imm <= 255)); // signed 9 bits + imm &= 0x1ff; // force into unsigned 9 bit representation + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= insEncodeIndexedOpt(id->idInsOpt()); // PP + code |= ((code_t)imm << 12); // iiiiiiiii + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_CZ_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - case IF_SVE_DA_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE propagate break from previous partition - { - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - - regNumber regm; - switch (ins) - { - case INS_sve_mov: - case INS_sve_movs: - regm = id->idReg3(); - break; - - case INS_sve_not: - case INS_sve_nots: - regm = id->idReg2(); - break; + case IF_LS_2D: // LS_2D .Q.............. ....ssnnnnnttttt Vt Rn + case IF_LS_2E: // LS_2E .Q.............. ....ssnnnnnttttt Vt Rn + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); - default: - regm = id->idReg4(); - } + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeVLSElemsize(elemsize); // ss + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - code |= insEncodeReg_P_19_to_16(regm); // MMMM dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_CZ_4A_A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - case IF_SVE_CZ_4A_L: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeReg_P_19_to_16(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_2F: // LS_2F .Q.............. xx.Sssnnnnnttttt Vt[] Rn + case IF_LS_2G: // LS_2G .Q.............. xx.Sssnnnnnttttt Vt[] Rn + elemsize = id->idOpSize(); + index = id->idSmallCns(); + code = emitInsCode(ins, fmt); + + code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - case IF_SVE_CZ_4A_K: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - code |= insEncodeReg_P_19_to_16(id->idReg1()); // DDDD dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DB_3A: // ................ ..gggg.NNNNMDDDD -- SVE partition break condition - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - code |= insEncodePredQualifier_4(id->idPredicateReg2Merge()); // M + case IF_LS_3A: // LS_3A .X.......X.mmmmm oooS..nnnnnttttt Rt Rn Rm ext(Rm) LSL {} + assert(insOptsLSExtend(id->idInsOpt())); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + } + else + { + code |= insEncodeDatasizeLS(code, id->idOpSize()); // .X.......X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + } + code |= insEncodeExtend(id->idInsOpt()); // ooo + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + if (id->idIsLclVar()) + { + code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm + } + else + { + code |= insEncodeReg3Scale(id->idReg3Scaled()); // S + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + } dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DB_3B: // ................ ..gggg.NNNN.DDDD -- SVE partition break condition - case IF_SVE_DC_3A: // ................ ..gggg.NNNN.MMMM -- SVE propagate break to next partition - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + case IF_LS_3B: // LS_3B X............... .aaaaannnnnddddd Rd Ra Rn + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + code |= insEncodeReg_Va(id->idReg2()); // aaaaa + } + else + { + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + code |= insEncodeReg_Ra(id->idReg2()); // aaaaa + } + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DD_2A: // ................ .......gggg.DDDD -- SVE predicate first active - case IF_SVE_DG_2A: // ................ .......gggg.DDDD -- SVE predicate read from FFR (predicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // gggg + case IF_LS_3C: // LS_3C X......PP.iiiiii iaaaaannnnnddddd Rd Ra Rn imm(im7,sh) + assert(insOptsNone(id->idInsOpt()) || insOptsIndexed(id->idInsOpt())); + imm = emitGetInsSC(id); + assert((imm >= -64) && (imm <= 63)); // signed 7 bits + imm &= 0x7f; // force into unsigned 7 bit representation + code = emitInsCode(ins, fmt); + // Is the target a vector register? + if (isVectorRegister(id->idReg1())) + { + code &= 0x3FFFFFFF; // clear the size bits + code |= insEncodeDatasizeVPLS(code, id->idOpSize()); // XX + code |= insEncodeReg_Vt(id->idReg1()); // ttttt + code |= insEncodeReg_Va(id->idReg2()); // aaaaa + } + else + { + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rt(id->idReg1()); // ttttt + code |= insEncodeReg_Ra(id->idReg2()); // aaaaa + } + code |= insEncodePairIndexedOpt(ins, id->idInsOpt()); // PP + code |= ((code_t)imm << 15); // iiiiiiiii + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DE_1A: // ........xx...... ......ppppp.DDDD -- SVE predicate initialize - code = emitInsCodeSve(ins, fmt); - code |= insEncodeSvePattern(id->idSvePattern()); // ppppp - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_LS_3D: // LS_3D .X.......X.mmmmm ......nnnnnttttt Wm Rt Rn + code = emitInsCode(ins, fmt); + // Arm64 store exclusive unpredictable cases + assert(id->idReg1() != id->idReg2()); + assert(id->idReg1() != id->idReg3()); + code |= insEncodeDatasizeLS(code, id->idOpSize()); // X + code |= insEncodeReg_Rm(id->idReg1()); // mmmmm + code |= insEncodeReg_Rt(id->idReg2()); // ttttt + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_8_to_5(id->idReg2()); // VVVV - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_LS_3E: // LS_3E .X.........mmmmm ......nnnnnttttt Rm Rt Rn ARMv8.1 LSE Atomics + code = emitInsCode(ins, fmt); + code |= insEncodeDatasizeLS(code, id->idOpSize()); // X + code |= insEncodeReg_Rm(id->idReg1()); // mmmmm + code |= insEncodeReg_Rt(id->idReg2()); // ttttt + code |= insEncodeReg_Rn(id->idReg3()); // nnnnn dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) - case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - dst += emitOutput_Instr(dst, code); - break; + // Some instructions with this encoding return their result in the + // second operand register instead of the first so we special case + // the GC update here and skip the common path down below. + if (emitInsDestIsOp2(ins)) + { + if (id->idGCref() != GCT_NONE) + { + emitGCregLiveUpd(id->idGCref(), id->idReg2(), dst); + } + else + { + emitGCregDeadUpd(id->idReg2(), dst); + } - case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_13_to_10(id->idReg1()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - dst += emitOutput_Instr(dst, code); - break; + goto SKIP_GC_UPDATE; + } - case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg - code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GA_2A: // ............iiii ......nnnn.ddddd -- SME2 multi-vec shift narrow - imm = emitGetInsSC(id); - assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); - assert(emitInsIsVectorRightShift(id->idIns())); - assert(isValidVectorShiftAmount(imm, EA_4BYTE, /* rightShift */ true)); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeVectorShift(EA_4BYTE, true /* right-shift */, imm); // iiii - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_6_Times_Two(id->idReg2()); // nnnn - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_3F: // LS_3F .Q.........mmmmm ....ssnnnnnttttt Vt Rn Rm + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); - case IF_SVE_DL_2A: // ........xx...... .....l.NNNNddddd -- SVE predicate count (predicate-as-counter) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeVectorLengthSpecifier(id); // l - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeVLSElemsize(elemsize); // ss + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - case IF_SVE_DM_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec register by predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DN_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec vector by predicate count - case IF_SVE_DP_2A: // ........xx...... .......MMMMddddd -- SVE saturating inc/dec vector by predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + case IF_LS_3G: // LS_3G .Q.........mmmmm ...Sssnnnnnttttt Vt[] Rn Rm + elemsize = id->idOpSize(); + index = id->idSmallCns(); + code = emitInsCode(ins, fmt); - case IF_SVE_DO_2A: // ........xx...... .....X.MMMMddddd -- SVE saturating inc/dec register by predicate count - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM - code |= insEncodeVLSElemsize(id->idOpSize()); // X - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + code |= insEncodeVLSIndex(elemsize, index); // Q xx S ss + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vt(id->idReg1()); // ttttt - case IF_SVE_DQ_0A: // ................ ................ -- SVE FFR initialise - code = emitInsCodeSve(ins, fmt); dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DR_1A: // ................ .......NNNN..... -- SVE FFR write from predicate - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_8_to_5(id->idReg1()); // NNNN + case IF_DI_1A: // DI_1A X.......shiiiiii iiiiiinnnnn..... Rn imm(i12,sh) + assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); + imm = emitGetInsSC(id); + assert(isValidUimm<12>(imm)); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftImm12(id->idInsOpt()); // sh + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DS_2A: // .........x.mmmmm ......nnnnn..... -- SVE conditionally terminate scalars - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_R_9_to_5(id->idReg1()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm - code |= insEncodeSveElemsize_R_22(id->idOpSize()); // x + case IF_DI_1B: // DI_1B X........hwiiiii iiiiiiiiiiiddddd Rd imm(i16,hw) + imm = emitGetInsSC(id); + assert(isValidImmHWVal(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= ((code_t)imm << 5); // hwiiiii iiiiiiiiiii + code |= insEncodeReg_Rd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FZ_2A: // ................ ......nnnn.ddddd -- SME2 multi-vec extract narrow - case IF_SVE_HG_2A: // ................ ......nnnn.ddddd -- SVE2 FP8 downconverts - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_6(id->idReg2()); // nnnn + case IF_DI_1C: // DI_1C X........Nrrrrrr ssssssnnnnn..... Rn imm(N,r,s) + imm = emitGetInsSC(id); + assert(isValidImmNRS(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GD_2A: // .........x.xx... ......nnnnnddddd -- SVE2 saturating extract narrow - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - // Bit 23 should not be set by below call - assert(insOptsScalableWide(id->idInsOpt())); - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx - // x + case IF_DI_1D: // DI_1D X........Nrrrrrr ssssss.....ddddd Rd imm(N,r,s) + imm = emitGetInsSC(id); + assert(isValidImmNRS(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<20, 16>(emitGetInsSC(id)); // iii - // Bit 23 should not be set by below call - assert(insOptsScalableWide(id->idInsOpt())); - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx - // x - dst += emitOutput_Instr(dst, code); + case IF_DI_1E: // DI_1E .ii.....iiiiiiii iiiiiiiiiiiddddd Rd simm21 + case IF_LARGEADR: + assert(insOptsNone(id->idInsOpt())); + if (id->idIsReloc()) + { + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + dst += emitOutput_Instr(dst, code); + emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADR_PAGE21 + : IMAGE_REL_ARM64_PAGEBASE_REL21); + } + else + { + // Local jmp/load case which does not need a relocation. + assert(id->idIsBound()); + dst = emitOutputLJ(ig, dst, id); + } + sz = sizeof(instrDescJmp); break; - case IF_SVE_GB_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift right narrow - // Bit 23 should not be set by call to insEncodeSveElemsize_tszh_23_tszl_20_to_19, - // nor should we pass INS_OPTS_SCALABLE_D to insGetImmDiff. - assert(insOptsScalableWide(id->idInsOpt())); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<20, 16>(insGetImmDiff(emitGetInsSC(id), id->idInsOpt())); // iii - code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx - // x - dst += emitOutput_Instr(dst, code); + case IF_DI_1F: // DI_1F X..........iiiii cccc..nnnnn.nzcv Rn imm5 nzcv cond + imm = emitGetInsSC(id); + assert(isValidImmCondFlagsImm5(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= ((code_t)cfi.imm5 << 16); // iiiii + code |= insEncodeFlags(cfi.flags); // nzcv + code |= insEncodeCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_FV_2A: // ........xx...... .....rmmmmmddddd -- SVE2 complex integer add - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<10, 10>(emitGetInsSC(id)); // r - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DI_2A: // DI_2A X.......shiiiiii iiiiiinnnnnddddd Rd Rn imm(i12,sh) + assert(insOptsNone(id->idInsOpt()) || insOptsLSL12(id->idInsOpt())); + imm = emitGetInsSC(id); + assert(isValidUimm<12>(imm)); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftImm12(id->idInsOpt()); // sh + code |= ((code_t)imm << 10); // iiiiiiiiiiii + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); - break; - case IF_SVE_FY_3A: // .........x.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long with carry - { - // Size encoding: 1 if INS_OPTS_SCALABLE_D, 0 if INS_OPTS_SCALABLE_S - const ssize_t sizeEncoding = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? 1 : 0; - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<22, 22>(sizeEncoding); // x - dst += emitOutput_Instr(dst, code); + if (id->idIsReloc()) + { + assert(sz == sizeof(instrDesc)); + assert(id->idAddr()->iiaAddr != nullptr); + emitRecordRelocation(odst, id->idAddr()->iiaAddr, id->idIsTlsGD() ? IMAGE_REL_AARCH64_TLSDESC_ADD_LO12 + : IMAGE_REL_ARM64_PAGEOFFSET_12A); + } break; - } - case IF_SVE_GK_2A: // ................ ......mmmmmddddd -- SVE2 crypto destructive binary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + case IF_DI_2B: // DI_2B X.........Xnnnnn ssssssnnnnnddddd Rd Rn imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // Reg2 also in mmmmm + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GL_1A: // ................ ...........ddddd -- SVE2 crypto unary operations - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + case IF_DI_2C: // DI_2C X........Nrrrrrr ssssssnnnnnddddd Rd Rn imm(N,r,s) + imm = emitGetInsSC(id); + assert(isValidImmNRS(imm, id->idOpSize())); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DT_3A: // ........xx.mmmmm ...X..nnnnn.DDDD -- SVE integer compare scalar count and limit - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= (id->idOpSize() == EA_8BYTE) ? (1 << 12) : 0; // X - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + case IF_DI_2D: // DI_2D X........Nrrrrrr ssssssnnnnnddddd Rd Rn imr, imms (N,r,s) + if (ins == INS_asr || ins == INS_lsl || ins == INS_lsr) + { + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); - case IF_SVE_DW_2A: // ........xx...... ......iiNNN.DDDD -- SVE extract mask predicate from predicate-as-counter - case IF_SVE_DW_2B: // ........xx...... .......iNNN.DDDD -- SVE extract mask predicate from predicate-as-counter - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_7_to_5(id->idReg2()); // NNN - code |= insEncodeUimm<9, 8>(emitGetInsSC(id)); // ii (or i) - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // Shift immediates are aliases of the SBFM/UBFM instructions + // that actually take 2 registers and 2 constants, + // Since we stored the shift immediate value + // we need to calculate the N,R and S values here. - case IF_SVE_DX_3A: // ........xx.mmmmm ......nnnnn.DDD. -- SVE integer compare scalar count and limit (predicate - // pair) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_1(id->idReg1()); // DDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + bitMaskImm bmi; + bmi.immNRS = 0; - case IF_SVE_DY_3A: // ........xx.mmmmm ..l...nnnnn..DDD -- SVE integer compare scalar count and limit - // (predicate-as-counter) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeVectorLengthSpecifier(id); // l - code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + bmi.immN = (size == EA_8BYTE) ? 1 : 0; + bmi.immR = imm; + bmi.immS = (size == EA_8BYTE) ? 0x3f : 0x1f; - case IF_SVE_DZ_1A: // ........xx...... .............DDD -- sve_int_pn_ptrue - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + // immR and immS are now set correctly for INS_asr and INS_lsr + // but for INS_lsl we have to adjust the values for immR and immS + // + if (ins == INS_lsl) + { + bmi.immR = -imm & bmi.immS; + bmi.immS = bmi.immS - imm; + } - case IF_SVE_EA_1A: // ........xx...... ...iiiiiiiiddddd -- SVE broadcast floating-point immediate (unpredicated) - case IF_SVE_ED_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer min/max immediate (unpredicated) - case IF_SVE_EE_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer multiply immediate (unpredicated) - { - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; - } + // setup imm with the proper 13 bit value N:R:S + // + imm = bmi.immNRS; + } + else + { + // The other instructions have already have encoded N,R and S values + imm = emitGetInsSC(id); + } + assert(isValidImmNRS(imm, id->idOpSize())); - case IF_SVE_FA_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) - case IF_SVE_FB_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) - case IF_SVE_FC_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) - { - const ssize_t imm = emitGetInsSC(id); - const ssize_t rot = (imm & 0b11); - const ssize_t index = (imm >> 2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 10>(rot); // rr - code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm - code |= insEncodeUimm<20, 19>(index); // ii + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 10); // Nrrrrrrssssss + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_EJ_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer dot product - case IF_SVE_EK_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // rr - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_DR_1D: // DR_1D X............... cccc.......ddddd Rd cond + imm = emitGetInsSC(id); + assert(isValidImmCond(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeInvertedCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_FA_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) - case IF_SVE_FB_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) - case IF_SVE_FC_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) - case IF_SVE_GV_3A: // ...........immmm ....rrnnnnnddddd -- SVE floating-point complex multiply-add (indexed) - { - const ssize_t imm = emitGetInsSC(id); - const ssize_t rot = (imm & 0b11); - const ssize_t index = (imm >> 2); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm - code |= insEncodeUimm<11, 10>(rot); // rr - - // index is encoded at bit location 20; - // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 - code |= insEncodeUimm<20, 19>(index << 1); // i + case IF_DR_2A: // DR_2A X..........mmmmm ......nnnnn..... Rn Rm + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - } - case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) - case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) + case IF_DR_2B: // DR_2B X.......sh.mmmmm ssssssnnnnn..... Rn Rm {LSL,LSR,ASR,ROR} imm(0-63) + code = emitInsCode(ins, fmt); imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx - code |= insEncodeImm8_12_to_5(imm); // iiiiiiii - code |= (id->idHasShift() ? 0x2000 : 0); // h + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftType(id->idInsOpt()); // sh + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_EB_1B: // ........xx...... ...........ddddd -- SVE broadcast integer immediate (unpredicated) - // ins is MOV for this encoding, as it is the preferred disassembly, so pass FMOV to emitInsCodeSve - code = emitInsCodeSve(INS_sve_fmov, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DR_2C: // DR_2C X..........mmmmm ooosssnnnnn..... Rn Rm ext(Rm) LSL imm(0-4) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert((imm >= 0) && (imm <= 4)); // imm [0..4] + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeExtend(id->idInsOpt()); // ooo + code |= insEncodeExtendScale(imm); // sss + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_DU_3A: // ........xx.mmmmm ......nnnnn.DDDD -- SVE pointer conflict compare - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); + case IF_DR_2D: // DR_2D X..........nnnnn cccc..nnnnnddddd Rd Rn cond + imm = emitGetInsSC(id); + assert(isValidImmCond(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm + code |= insEncodeInvertedCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } break; - case IF_SVE_DV_4A: // ........ix.xxxvv ..NNNN.MMMM.DDDD -- SVE broadcast predicate element - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN - code |= insEncodeReg_P_8_to_5(id->idReg3()); // MMMM - code |= insEncodeReg_R_17_to_16(id->idReg4()); // vv - code |= insEncodeSveElemsize_tszh_tszl_and_imm(id->idInsOpt(), emitGetInsSC(id)); // ix xx + case IF_DR_2E: // DR_2E X..........mmmmm ...........ddddd Rd Rm + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + case IF_DR_2F: // DR_2F X.......sh.mmmmm ssssss.....ddddd Rd Rm {LSL,LSR,ASR} imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeShiftType(id->idInsOpt()); // sh + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - switch (id->idInsOpt()) + case IF_DR_2G: // DR_2G X............... .....xnnnnnddddd Rd Rn + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + if (ins == INS_rev) { - case INS_OPTS_H_TO_S: - code |= (1 << 16); - break; - case INS_OPTS_H_TO_D: - code |= (1 << 22) | (1 << 16); - break; - case INS_OPTS_S_TO_H: - break; - case INS_OPTS_S_TO_D: - code |= (1 << 22) | (3 << 16); - break; - case INS_OPTS_D_TO_H: - code |= (1 << 22); - break; - case INS_OPTS_D_TO_S: - code |= (1 << 22) | (1 << 17); - break; - default: - unreached(); + if (size == EA_8BYTE) + { + code |= 0x00000400; // x - bit at location 10 + } } + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HO_3C: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + case IF_DR_2H: // DR_2H X........X...... ......nnnnnddddd Rd Rn + code = emitInsCode(ins, fmt); + code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + case IF_DR_2I: // DR_2I X..........mmmmm cccc..nnnnn.nzcv Rn Rm nzcv cond + imm = emitGetInsSC(id); + assert(isValidImmCondFlags(imm)); + { + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rn(id->idReg1()); // nnnnn + code |= insEncodeReg_Rm(id->idReg2()); // mmmmm + code |= insEncodeFlags(cfi.flags); // nzcv + code |= insEncodeCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); + } + break; - switch (id->idInsOpt()) + case IF_DR_3A: // DR_3A X..........mmmmm ......nnnnnmmmmm Rd Rn Rm + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + if (id->idIsLclVar()) { - case INS_OPTS_SCALABLE_H: - code |= (1 << 22) | (1 << 17); - break; - case INS_OPTS_H_TO_S: - code |= (1 << 22) | (1 << 18); - break; - case INS_OPTS_H_TO_D: - code |= (1 << 22) | (3 << 17); - break; - case INS_OPTS_SCALABLE_S: - code |= (1 << 23) | (1 << 18); - break; - case INS_OPTS_S_TO_D: - code |= (3 << 22) | (1 << 18); - break; - case INS_OPTS_D_TO_S: - code |= (3 << 22); - break; - case INS_OPTS_SCALABLE_D: - code |= (3 << 22) | (3 << 17); - break; - default: - unreached(); - break; + code |= insEncodeReg_Rm(codeGen->rsGetRsvdReg()); // mmmmm + } + else + { + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm } + dst += emitOutput_Instr(dst, code); + break; + case IF_DR_3B: // DR_3B X.......sh.mmmmm ssssssnnnnnddddd Rd Rn Rm {LSL,LSR,ASR} imm(0-63) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeShiftType(id->idInsOpt()); // sh + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + case IF_DR_3C: // DR_3C X..........mmmmm ooosssnnnnnddddd Rd Rn Rm ext(Rm) LSL imm(0-4) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + assert((imm >= 0) && (imm <= 4)); // imm [0..4] + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeExtend(id->idInsOpt()); // ooo + code |= insEncodeExtendScale(imm); // sss + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; - switch (id->idInsOpt()) + case IF_DR_3D: // DR_3D X..........mmmmm cccc..nnnnnddddd Rd Rn Rm cond + imm = emitGetInsSC(id); + assert(isValidImmCond(imm)); { - case INS_OPTS_SCALABLE_H: - code |= (1 << 22) | (1 << 17); - break; - case INS_OPTS_S_TO_H: - code |= (1 << 22) | (1 << 18); - break; - case INS_OPTS_SCALABLE_S: - code |= (1 << 23) | (1 << 18); - break; - case INS_OPTS_S_TO_D: - code |= (1 << 23) | (1 << 22); - break; - case INS_OPTS_D_TO_H: - code |= (1 << 22) | (3 << 17); - break; - case INS_OPTS_D_TO_S: - code |= (3 << 22) | (1 << 18); - break; - case INS_OPTS_SCALABLE_D: - code |= (3 << 22) | (3 << 17); - break; - default: - unreached(); - break; + condFlagsImm cfi; + cfi.immCFVal = (unsigned)imm; + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeCond(cfi.cond); // cccc + dst += emitOutput_Instr(dst, code); } - - dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) - case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) - case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus - // immediate) - case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) - case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) - case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus - // immediate) - case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus - // immediate) - case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus - // immediate) - case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus - // immediate) - case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus - // immediate) - case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus - // immediate) - case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) - case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus - // immediate) - case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus - // immediate) - case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) + case IF_DR_3E: // DR_3E X........X.mmmmm ssssssnnnnnddddd Rd Rn Rm imm(0-63) + code = emitInsCode(ins, fmt); imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + assert(isValidImmShift(imm, id->idOpSize())); + code |= insEncodeDatasizeBF(code, id->idOpSize()); // X........X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeShiftCount(imm, id->idOpSize()); // ssssss + dst += emitOutput_Instr(dst, code); + break; - switch (ins) - { - case INS_sve_ld2b: - case INS_sve_ld2h: - case INS_sve_ld2w: - case INS_sve_ld2d: - case INS_sve_ld2q: - case INS_sve_st2b: - case INS_sve_st2h: - case INS_sve_st2w: - case INS_sve_st2d: - case INS_sve_st2q: - code |= insEncodeSimm4_MultipleOf2_19_to_16(imm); // iiii - break; + case IF_DR_4A: // DR_4A X..........mmmmm .aaaaannnnnmmmmm Rd Rn Rm Ra + code = emitInsCode(ins, fmt); + code |= insEncodeDatasize(id->idOpSize()); // X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeReg_Ra(id->idReg4()); // aaaaa + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ld3b: - case INS_sve_ld3h: - case INS_sve_ld3w: - case INS_sve_ld3d: - case INS_sve_ld3q: - case INS_sve_st3b: - case INS_sve_st3h: - case INS_sve_st3w: - case INS_sve_st3d: - case INS_sve_st3q: - code |= insEncodeSimm4_MultipleOf3_19_to_16(imm); // iiii - break; + case IF_DV_1A: // DV_1A .........X.iiiii iii........ddddd Vd imm8 (fmov - immediate scalar) + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= ((code_t)imm << 13); // iiiii iii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + dst += emitOutput_Instr(dst, code); + break; - case INS_sve_ld4b: - case INS_sve_ld4h: - case INS_sve_ld4w: - case INS_sve_ld4d: - case INS_sve_ld4q: - case INS_sve_st4b: - case INS_sve_st4h: - case INS_sve_st4w: - case INS_sve_st4d: - case INS_sve_st4q: - code |= insEncodeSimm4_MultipleOf4_19_to_16(imm); // iiii + case IF_DV_1B: // DV_1B .QX..........iii cmod..iiiiiddddd Vd imm8 (immediate vector) + imm = emitGetInsSC(id) & 0x0ff; + immShift = (emitGetInsSC(id) & 0x700) >> 8; + elemsize = optGetElemsize(id->idInsOpt()); + cmode = 0; + switch (elemsize) + { // cmode + case EA_1BYTE: + cmode = 0xE; // 1110 break; - - case INS_sve_ld1rqb: - case INS_sve_ld1rqd: - case INS_sve_ld1rqh: - case INS_sve_ld1rqw: - code |= insEncodeSimm4_MultipleOf16_19_to_16(imm); // iiii + case EA_2BYTE: + cmode = 0x8; + cmode |= (immShift << 1); // 10x0 break; - - case INS_sve_ld1rob: - case INS_sve_ld1rod: - case INS_sve_ld1roh: - case INS_sve_ld1row: - code |= insEncodeSimm4_MultipleOf32_19_to_16(imm); // iiii + case EA_4BYTE: + if (immShift < 4) + { + cmode = 0x0; + cmode |= (immShift << 1); // 0xx0 + } + else // MSL + { + cmode = 0xC; + if (immShift & 2) + cmode |= 1; // 110x + } + break; + case EA_8BYTE: + cmode = 0xE; // 1110 break; - default: - code |= insEncodeSimm<19, 16>(imm); // iiii + unreached(); break; } - if (canEncodeSveElemsize_dtype(ins)) + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + if ((ins == INS_fmov) || (ins == INS_movi)) { - if (ins == INS_sve_ld1w) - { - code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); - } - else + if (elemsize == EA_8BYTE) { - code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); + code |= 0x20000000; // X } } - + if (ins != INS_fmov) + { + assert((cmode >= 0) && (cmode <= 0xF)); + code |= (cmode << 12); // cmod + } + code |= (((code_t)imm >> 5) << 16); // iii + code |= (((code_t)imm & 0x1f) << 5); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JD_4A: // .........xxmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_1C: // DV_1C .........X...... ......nnnnn..... Vn #0.0 (fcmp - with zero) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vn(id->idReg1()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JD_4B: // ..........xmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + case IF_DV_2A: // DV_2A .Q.......X...... ......nnnnnddddd Vd Vn (fabs, fcvt - vector) + case IF_DV_2R: // DV_2R .Q.......X...... ......nnnnnddddd Sd Vn (fmaxnmv, fmaxv, fminnmv, fminv) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + if ((ins == INS_fcvtl) || (ins == INS_fcvtl2) || (ins == INS_fcvtn) || (ins == INS_fcvtn2)) + { + // fcvtl{2} and fcvtn{2} encode the element size as + // esize = 16 << UInt(sz) + if (elemsize == EA_4BYTE) + { + code |= 0x00400000; // X + } + else + { + assert(elemsize == EA_2BYTE); + } + } + else + { + code |= insEncodeFloatElemsize(elemsize); // X + } + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JJ_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // // offsets) - case IF_SVE_JJ_4A_C: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4A_D: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JK_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled - // offsets) - case IF_SVE_JK_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit - // unscaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm - - switch (id->idInsOpt()) + case IF_DV_2B: // DV_2B .Q.........iiiii ......nnnnnddddd Rd Vn[] (umov/smov - to general) + elemsize = id->idOpSize(); + index = emitGetInsSC(id); + datasize = (elemsize == EA_8BYTE) ? EA_16BYTE : EA_8BYTE; + if (ins == INS_smov) { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 14); // h - break; - - default: - break; + datasize = EA_16BYTE; } - + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(datasize); // Q + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JN_3A: // .........xx.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm<19, 16>(imm); // iiii - code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2C: // DV_2C .Q.........iiiii ......nnnnnddddd Vd Rn (dup/ins - vector from general) + if (ins == INS_dup) + { + datasize = id->idOpSize(); + elemsize = optGetElemsize(id->idInsOpt()); + index = 0; + } + else // INS_ins + { + datasize = EA_16BYTE; + elemsize = id->idOpSize(); + index = emitGetInsSC(id); + } + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(datasize); // Q + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JN_3B: // ..........x.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSimm<19, 16>(imm); // iiii - code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + case IF_DV_2D: // DV_2D .Q.........iiiii ......nnnnnddddd Vd Vn[] (dup - vector) + index = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HW_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4A_B: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_IU_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2E: // DV_2E ...........iiiii ......nnnnnddddd Vd Vn[] (dup - scalar) + index = emitGetInsSC(id); + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 22); // h - break; + case IF_DV_2F: // DV_2F ...........iiiii .jjjj.nnnnnddddd Vd[] Vn[] (ins - element) + elemsize = id->idOpSize(); + imm = emitGetInsSC(id); + index = (imm >> 4) & 0xf; + index2 = imm & 0xf; + code = emitInsCode(ins, fmt); + code |= insEncodeVectorIndex(elemsize, index); // iiiii + code |= insEncodeVectorIndex2(elemsize, index2); // jjjj + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; - default: - break; - } + case IF_DV_2G: // DV_2G .........X...... ......nnnnnddddd Vd Vn (fmov, fcvtXX - register) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + case IF_DV_2H: // DV_2H X........X...... ......nnnnnddddd Rd Vn (fmov - to general) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HW_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - case IF_SVE_HW_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2I: // DV_2I X........X...... ......nnnnnddddd Vd Rn (fmov - from general) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // X X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus - // scalar) - case IF_SVE_IF_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus - // scalar) - case IF_SVE_IW_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit gather load (vector plus scalar) - case IF_SVE_IX_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit gather non-temporal load (vector plus - // scalar) - case IF_SVE_IY_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit scatter store (vector plus scalar) - case IF_SVE_IZ_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus - // scalar) - case IF_SVE_IZ_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus - // scalar) - case IF_SVE_JA_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit scatter non-temporal store (vector plus - // scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2J: // DV_2J ........SS.....D D.....nnnnnddddd Vd Vn (fcvt) + code = emitInsCode(ins, fmt); + code |= insEncodeConvertOpt(fmt, id->idInsOpt()); // SS DD + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IG_4A_D: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_IG_4A_E: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_IG_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_IG_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus - // scalar) - case IF_SVE_II_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) - case IF_SVE_IK_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IK_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IK_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IK_4A_I: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm - - if (canEncodeSveElemsize_dtype(ins)) - { - if (ins == INS_sve_ld1w) - { - code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); - } - else - { - code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); - } - } - + case IF_DV_2K: // DV_2K .........X.mmmmm ......nnnnn..... Vn Vm (fcmp) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vn(id->idReg1()); // nnnnn + code |= insEncodeReg_Vm(id->idReg2()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IG_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus scalar) - case IF_SVE_II_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) - case IF_SVE_II_4A_B: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) - case IF_SVE_IK_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) - case IF_SVE_IN_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus scalar) - case IF_SVE_IP_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus scalar) - case IF_SVE_IR_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus - // scalar) - case IF_SVE_IT_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (scalar plus scalar) - case IF_SVE_JB_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus - // scalar) - case IF_SVE_JC_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (scalar plus scalar) - case IF_SVE_JD_4C: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - case IF_SVE_JD_4C_A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) - case IF_SVE_JF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus - // scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2L: // DV_2L ........XX...... ......nnnnnddddd Vd Vn (abs, neg - scalar) + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IU_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4B_B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_IU_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked - // scaled offsets) - case IF_SVE_JJ_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4B_C: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JJ_4B_E: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled - // offsets) - case IF_SVE_JK_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_2M: // DV_2M .Q......XX...... ......nnnnnddddd Vd Vn (abs, neg - vector) + case IF_DV_2T: // DV_2T .Q......XX...... ......nnnnnddddd Sd Vn (addv, saddlv, smaxv, sminv, uaddlv, + // umaxv, uminv) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GP_3A: // ........xx.....r ...gggmmmmmddddd -- SVE floating-point complex add (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeSveImm90_or_270_rot(imm); // r - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2N: // DV_2N .........iiiiiii ......nnnnnddddd Vd Vn imm (shift - scalar) + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GT_4A: // ........xx.mmmmm .rrgggnnnnnddddd -- SVE floating-point complex multiply-add (predicated) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm - code |= insEncodeSveImm0_to_270_rot(imm); // rr - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2O: // DV_2O .Q.......iiiiiii ......nnnnnddddd Vd Vn imm (shift - vector) + imm = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeVectorShift(elemsize, emitInsIsVectorRightShift(ins), imm); // iiiiiii + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HI_3A: // ........xx...... ...gggnnnnn.DDDD -- SVE floating-point compare with zero - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2P: // DV_2P ............... ......nnnnnddddd Vd Vn (aes*, sha1su1) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HM_2A: // ........xx...... ...ggg....iddddd -- SVE floating-point arithmetic with immediate - // (predicated) - { - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeSveSmallFloatImm(imm); // i - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - } + case IF_DV_2Q: // DV_2Q .........X...... ......nnnnnddddd Vd Vn (faddp, fmaxnmp, fmaxp, fminnmp, + // fminp - scalar) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HN_2A: // ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<18, 16>(imm); // iii - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2S: // DV_2S ........XX...... ......nnnnnddddd Sd Vn (addp - scalar) + elemsize = optGetElemsize(id->idInsOpt()); + code = emitInsCode(ins, fmt); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HP_3A: // .............xx. ...gggnnnnnddddd -- SVE floating-point convert to integer - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize_18_to_17(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_2U: // DV_2U ................ ......nnnnnddddd Sd Sn (sha1h) + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HU_4B: // ...........mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + case IF_DV_3A: // DV_3A .Q......XX.mmmmm ......nnnnnddddd Vd Vn Vm (vector) + code = emitInsCode(ins, fmt); + elemsize = optGetElemsize(id->idInsOpt()); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HV_4A: // ........xx.aaaaa ...gggmmmmmddddd -- SVE floating-point multiply-accumulate writing - // multiplicand - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm - code |= insEncodeReg_V_20_to_16(id->idReg4()); // aaaaa - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + case IF_DV_3AI: // DV_3AI .Q......XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_ID_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE load predicate register - case IF_SVE_JG_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE store predicate register - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_3_to_0(id->idReg1()); // TTTT - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii - // iiiiii + case IF_DV_3B: // DV_3B .Q.......X.mmmmm ......nnnnnddddd Vd Vn Vm (vector) + code = emitInsCode(ins, fmt); + elemsize = optGetElemsize(id->idInsOpt()); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IE_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE load vector register - case IF_SVE_JH_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE store vector register - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii - // iiiiii + case IF_DV_3BI: // DV_3BI .Q.......XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (vector by element) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = optGetElemsize(id->idInsOpt()); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeFloatIndex(elemsize, imm); // L H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm<23, 22>(imm); // ii + case IF_DV_3C: // DV_3C .Q.........mmmmm ......nnnnnddddd Vd Vn Vm (vector) + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm3h3l_23_to_22_and_12(imm); // ii - // i + case IF_DV_3D: // DV_3D .........X.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) + code = emitInsCode(ins, fmt); + code |= insEncodeFloatElemsize(id->idOpSize()); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit - // element size - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= insEncodeUimm1_23(imm); // i + case IF_DV_3DI: // DV_3DI .........XLmmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeFloatIndex(elemsize, imm); // L H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit - // scaled offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo - - switch (id->idInsOpt()) - { - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - code |= (1 << 22); // h - break; - - default: - break; - } + case IF_DV_3E: // DV_3E ........XX.mmmmm ......nnnnnddddd Vd Vn Vm (scalar) + code = emitInsCode(ins, fmt); + elemsize = id->idOpSize(); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + case IF_DV_3EI: // DV_3EI ........XXLMmmmm ....H.nnnnnddddd Vd Vn Vm[] (scalar by element) + code = emitInsCode(ins, fmt); + imm = emitGetInsSC(id); + elemsize = id->idOpSize(); + assert(isValidVectorIndex(EA_16BYTE, elemsize, imm)); + code |= insEncodeElemsize(elemsize); // XX + code |= insEncodeVectorIndexLMH(elemsize, imm); // LM H + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled - // offsets) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo + case IF_DV_3F: // DV_3F ...........mmmmm ......nnnnnddddd Vd Vn Vm (vector) - source dest regs overlap + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm dst += emitOutput_Instr(dst, code); break; - case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm - code |= id->idSvePrfop(); // oooo + case IF_DV_3G: // DV_3G .Q.........mmmmm .iiii.nnnnnddddd Vd Vn Vm imm (vector) + imm = emitGetInsSC(id); + code = emitInsCode(ins, fmt); + code |= insEncodeVectorsize(id->idOpSize()); // Q + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm + code |= ((code_t)imm << 11); // iiii + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vd(id->idReg1()); // ddddd dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= id->idSvePrfop(); // oooo + case IF_DV_4A: // DV_4A .........X.mmmmm .aaaaannnnnddddd Vd Va Vn Vm (scalar) + code = emitInsCode(ins, fmt); + elemsize = id->idOpSize(); + code |= insEncodeFloatElemsize(elemsize); // X + code |= insEncodeReg_Vd(id->idReg1()); // ddddd + code |= insEncodeReg_Vn(id->idReg2()); // nnnnn + code |= insEncodeReg_Vm(id->idReg3()); // mmmmm + code |= insEncodeReg_Va(id->idReg4()); // aaaaa + dst += emitOutput_Instr(dst, code); + break; - if (id->idInsOpt() == INS_OPTS_SCALABLE_D) + case IF_SN_0A: // SN_0A ................ ................ + { + bool skipIns = false; +#if FEATURE_LOOP_ALIGN + if (id->idIns() == INS_align) { - code |= (1 << 30); // set bit '30' to make it a double-word - } + // IG can be marked as not needing alignment after emitting align instruction. + // Alternatively, there are fewer align instructions needed than emitted. + // If that is the case, skip outputting alignment. + if (!ig->endsWithAlignInstr() || id->idIsEmptyAlign()) + { + skipIns = true; + } - switch (ins) - { - case INS_sve_prfh: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; +#ifdef DEBUG + if (!ig->endsWithAlignInstr()) + { + // Validate if the state is correctly updated + assert(id->idIsEmptyAlign()); + } +#endif + sz = sizeof(instrDescAlign); + ins = INS_nop; - case INS_sve_prfw: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; +#ifdef DEBUG + // Under STRESS_EMITTER, if this is the 'align' before the 'jmp' instruction, + // then add "bkpt" instruction. + instrDescAlign* alignInstr = (instrDescAlign*)id; - case INS_sve_prfd: - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii - break; + if (emitComp->compStressCompile(Compiler::STRESS_EMITTER, 50) && alignInstr->isPlacedAfterJmp && + !skipIns) + { + // There is no good way to squeeze in "bkpt" as well as display it + // in the disassembly because there is no corresponding instrDesc for + // it. As such, leave it as is, the "0xD43E0000" bytecode will be seen + // next to the nop instruction in disasm. + // e.g. D43E0000 align [4 bytes for IG07] + ins = INS_BREAKPOINT; + fmt = IF_SI_0A; + } +#endif + } +#endif // FEATURE_LOOP_ALIGN - default: - assert(ins == INS_sve_prfb); + if (!skipIns) + { + code = emitInsCode(ins, fmt); + dst += emitOutput_Instr(dst, code); } - dst += emitOutput_Instr(dst, code); + break; + } - case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm<20, 16>(imm); // iiiii - code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + case IF_SI_0A: // SI_0A ...........iiiii iiiiiiiiiii..... imm16 + imm = emitGetInsSC(id); + assert(isValidUimm<16>(imm)); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 5); // iiiii iiiiiiiiiii dst += emitOutput_Instr(dst, code); break; - case IF_SVE_HX_3A_E: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) - case IF_SVE_IV_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit gather load (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); - - switch (ins) - { - case INS_sve_ld1d: - case INS_sve_ldff1d: - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii - break; - - case INS_sve_ld1w: - case INS_sve_ld1sw: - case INS_sve_ldff1w: - case INS_sve_ldff1sw: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; - - default: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; - } - + case IF_SI_0B: // SI_0B ................ ....bbbb........ imm4 - barrier + imm = emitGetInsSC(id); + assert((imm >= 0) && (imm <= 15)); + code = emitInsCode(ins, fmt); + code |= ((code_t)imm << 8); // bbbb dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JL_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit scatter store (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii + case IF_SR_1A: // SR_1A ................ ...........ttttt Rt (dc zva, mrs) + assert(insOptsNone(id->idInsOpt())); + code = emitInsCode(ins, fmt); + code |= insEncodeReg_Rt(id->idReg1()); // ttttt dst += emitOutput_Instr(dst, code); break; - case IF_SVE_JI_3A_A: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit scatter store (vector plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn - code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); - - switch (ins) - { - case INS_sve_st1h: - code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii - break; + default: + dst = emitOutput_InstrSve(dst, id); + break; + } - case INS_sve_st1w: - code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii - break; + // Determine if any registers now hold GC refs, or whether a register that was overwritten held a GC ref. + // We assume here that "id->idGCref()" is not GC_NONE only if the instruction described by "id" writes a + // GC ref to register "id->idReg1()". (It may, apparently, also not be GC_NONE in other cases, such as + // for stores, but we ignore those cases here.) + if (emitInsMayWriteToGCReg(id)) // True if "id->idIns()" writes to a register than can hold GC ref. + { + // We assume that "idReg1" is the primary destination register for all instructions + assert(!emitInsDestIsOp2(ins)); + if (id->idGCref() != GCT_NONE) + { + emitGCregLiveUpd(id->idGCref(), id->idReg1(), dst); + } + else + { + emitGCregDeadUpd(id->idReg1(), dst); + } - default: - assert(ins == INS_sve_st1b); - code |= insEncodeUimm<20, 16>(imm); // iiiii - break; + if (emitInsMayWriteMultipleRegs(id)) + { + // INS_ldp etc... + // "idReg2" is the secondary destination register + if (id->idGCrefReg2() != GCT_NONE) + { + emitGCregLiveUpd(id->idGCrefReg2(), id->idReg2(), dst); } + else + { + emitGCregDeadUpd(id->idReg2(), dst); + } + } + } - dst += emitOutput_Instr(dst, code); - break; +SKIP_GC_UPDATE: + // Now we determine if the instruction has written to a (local variable) stack location, and either written a GC + // ref or overwritten one. + if (emitInsWritesToLclVarStackLoc(id) || emitInsWritesToLclVarStackLocPair(id)) + { + int varNum = id->idAddr()->iiaLclVar.lvaVarNum(); + unsigned ofs = AlignDown(id->idAddr()->iiaLclVar.lvaOffset(), TARGET_POINTER_SIZE); + bool FPbased; + int adr = emitComp->lvaFrameAddress(varNum, &FPbased); + if (id->idGCref() != GCT_NONE) + { + emitGCvarLiveUpd(adr + ofs, varNum, id->idGCref(), dst DEBUG_ARG(varNum)); + } + else + { + // If the type of the local is a gc ref type, update the liveness. + var_types vt; + if (varNum >= 0) + { + // "Regular" (non-spill-temp) local. + vt = var_types(emitComp->lvaTable[varNum].lvType); + } + else + { + TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum); + vt = tmpDsc->tdTempType(); + } + if (vt == TYP_REF || vt == TYP_BYREF) + { + emitGCvarDeadUpd(adr + ofs, dst DEBUG_ARG(varNum)); + } + } + if (emitInsWritesToLclVarStackLocPair(id)) + { + int varNum2 = varNum; + int adr2 = adr; + unsigned ofs2 = ofs; + unsigned ofs2Dist; - case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn - code |= id->idSvePrfop(); // oooo - code |= insEncodeSimm<21, 16>(imm); // iiiiii - dst += emitOutput_Instr(dst, code); - break; + if (id->idIsLclVarPair()) + { + bool FPbased2; - case IF_SVE_IC_3A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + emitLclVarAddr* lclVarAddr2 = emitGetLclVarPairLclVar2(id); + varNum2 = lclVarAddr2->lvaVarNum(); + ofs2 = lclVarAddr2->lvaOffset(); - switch (ins) + // If there are 2 GC vars in this instrDesc, get the 2nd variable + // that should be tracked. + adr2 = emitComp->lvaFrameAddress(varNum2, &FPbased2); + ofs2Dist = EA_SIZE_IN_BYTES(size); +#ifdef DEBUG + assert(FPbased == FPbased2); + if (FPbased) + { + assert(id->idReg3() == REG_FP); + } + else + { + assert(id->idReg3() == REG_SP); + } + assert(varNum2 != -1); +#endif // DEBUG + } + else { - case INS_sve_ld1rd: - code |= insEncodeUimm6_MultipleOf8_21_to_16(imm); // iiiiii - break; - - default: - assert(ins == INS_sve_ld1rsw); - code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii - break; + ofs2Dist = TARGET_POINTER_SIZE; + ofs2 += ofs2Dist; } - dst += emitOutput_Instr(dst, code); - break; - - case IF_SVE_IC_3A_A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - case IF_SVE_IC_3A_B: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - case IF_SVE_IC_3A_C: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt - code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg - code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn - code = insEncodeSveElemsize_dtypeh_dtypel(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + ofs2 = AlignDown(ofs2, ofs2Dist); - switch (ins) + if (id->idGCrefReg2() != GCT_NONE) + { +#ifdef DEBUG + if (id->idGCref() != GCT_NONE) + { + // If 1st register was a gc-var, then make sure the offset + // are correctly set for the 2nd register that is holding + // another gc-var. + assert((adr + ofs + ofs2Dist) == (adr2 + ofs2)); + } +#endif + emitGCvarLiveUpd(adr2 + ofs2, varNum2, id->idGCrefReg2(), dst DEBUG_ARG(varNum2)); + } + else { - case INS_sve_ld1rw: - code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii - break; - - case INS_sve_ld1rh: - case INS_sve_ld1rsh: - code |= insEncodeUimm6_MultipleOf2_21_to_16(imm); // iiiiii - break; - - default: - code |= insEncodeUimm<21, 16>(imm); // iiiiii - break; + // If the type of the local is a gc ref type, update the liveness. + var_types vt; + if (varNum2 >= 0) + { + // "Regular" (non-spill-temp) local. + vt = var_types(emitComp->lvaTable[varNum2].lvType); + } + else + { + TempDsc* tmpDsc = codeGen->regSet.tmpFindNum(varNum2); + vt = tmpDsc->tdTempType(); + } + if (vt == TYP_REF || vt == TYP_BYREF) + { + emitGCvarDeadUpd(adr2 + ofs2, dst DEBUG_ARG(varNum2)); + } } + } + } - dst += emitOutput_Instr(dst, code); - break; +#ifdef DEBUG + /* Make sure we set the instruction descriptor size correctly */ - case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) - case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - dst += emitOutput_Instr(dst, code); - break; + size_t expected = emitSizeOfInsDsc(id); + assert(sz == expected); - case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_Rn(id->idReg2()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + if (emitComp->opts.disAsm || emitComp->verbose) + { + emitDispIns(id, false, dspOffs, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + } - case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator - case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements - case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements - case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx - dst += emitOutput_Instr(dst, code); - break; + if (emitComp->compDebugBreak) + { + // For example, set JitBreakEmitOutputInstr=a6 will break when this method is called for + // emitting instruction a6, (i.e. IN00a6 in jitdump). + if ((unsigned)JitConfig.JitBreakEmitOutputInstr() == id->idDebugOnlyInfo()->idNum) + { + assert(!"JitBreakEmitOutputInstr reached"); + } + } - case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) - case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert - case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsizeWithShift_tszh_tszl_imm3(id->idInsOpt(), imm, - emitInsIsVectorRightShift(ins)); // xx xxiii - dst += emitOutput_Instr(dst, code); - break; + // Output any delta in GC info. + if (EMIT_GC_VERBOSE || emitComp->opts.disasmWithGC) + { + emitDispGCInfoDelta(); + } +#else + if (emitComp->opts.disAsm) + { + size_t expected = emitSizeOfInsDsc(id); + assert(sz == expected); + emitDispIns(id, false, 0, true, emitCurCodeOffs(odst), *dp, (dst - *dp), ig); + } +#endif - case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn - code |= insEncodeSveElemsizeWithImmediate_i1_tsz(id->idInsOpt(), imm); // ixxxx - dst += emitOutput_Instr(dst, code); - break; + /* All instructions are expected to generate code */ - case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq - imm = emitGetInsSC(id); - code = emitInsCodeSve(ins, fmt); - code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd - code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm - code |= insEncodeUimm<19, 16>(imm); // iiii - dst += emitOutput_Instr(dst, code); - break; + assert(*dp != dst || id->idIsEmptyAlign()); - default: - assert(!"Unexpected format"); - break; - } + *dp = dst; - return dst; + return sz; } /*****************************************************************************/ @@ -20992,286 +16730,102 @@ void emitter::emitDispSmallFloatImm(ssize_t imm, instruction ins) */ void emitter::emitDispImmOptsLSL(ssize_t imm, bool hasShift, unsigned shiftAmount) { - if (!strictArmAsm && hasShift) - { - imm <<= shiftAmount; - } - emitDispImm(imm, false); - if (strictArmAsm && hasShift) - { - printf(", LSL #%u", shiftAmount); - } -} - -/***************************************************************************** - * - * Display an ARM64 condition code for the conditional instructions - */ -void emitter::emitDispCond(insCond cond) -{ - const static char* armCond[16] = {"eq", "ne", "hs", "lo", "mi", "pl", "vs", "vc", - "hi", "ls", "ge", "lt", "gt", "le", "AL", "NV"}; // The last two are invalid - unsigned imm = (unsigned)cond; - assert((0 <= imm) && (imm < ArrLen(armCond))); - printf(armCond[imm]); -} - -/***************************************************************************** - * - * Display an ARM64 flags for the conditional instructions - */ -void emitter::emitDispFlags(insCflags flags) -{ - const static char* armFlags[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", - "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; - unsigned imm = (unsigned)flags; - assert((0 <= imm) && (imm < ArrLen(armFlags))); - printf(armFlags[imm]); -} - -/***************************************************************************** - * - * Display an ARM64 'barrier' for the memory barrier instructions - */ -void emitter::emitDispBarrier(insBarrier barrier) -{ - const static char* armBarriers[16] = {"#0", "oshld", "oshst", "osh", "#4", "nshld", "nshst", "nsh", - "#8", "ishld", "ishst", "ish", "#12", "ld", "st", "sy"}; - unsigned imm = (unsigned)barrier; - assert((0 <= imm) && (imm < ArrLen(armBarriers))); - printf(armBarriers[imm]); -} - -/***************************************************************************** - * - * Prints the encoding for the Shift Type encoding - */ - -void emitter::emitDispShiftOpts(insOpts opt) -{ - if (opt == INS_OPTS_LSL) - printf(" LSL "); - else if (opt == INS_OPTS_LSR) - printf(" LSR "); - else if (opt == INS_OPTS_ASR) - printf(" ASR "); - else if (opt == INS_OPTS_ROR) - printf(" ROR "); - else if (opt == INS_OPTS_MSL) - printf(" MSL "); - else - assert(!"Bad value"); -} - -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding - */ - -void emitter::emitDispExtendOpts(insOpts opt) -{ - if (opt == INS_OPTS_UXTB) - printf("UXTB"); - else if (opt == INS_OPTS_UXTH) - printf("UXTH"); - else if (opt == INS_OPTS_UXTW) - printf("UXTW"); - else if (opt == INS_OPTS_UXTX) - printf("UXTX"); - else if (opt == INS_OPTS_SXTB) - printf("SXTB"); - else if (opt == INS_OPTS_SXTH) - printf("SXTH"); - else if (opt == INS_OPTS_SXTW) - printf("SXTW"); - else if (opt == INS_OPTS_SXTX) - printf("SXTX"); - else - assert(!"Bad value"); -} - -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding - */ - -void emitter::emitDispSveExtendOpts(insOpts opt) -{ - switch (opt) - { - case INS_OPTS_LSL: - printf("lsl"); - break; - - case INS_OPTS_UXTW: - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_D_UXTW: - printf("uxtw"); - break; - - case INS_OPTS_SXTW: - case INS_OPTS_SCALABLE_S_SXTW: - case INS_OPTS_SCALABLE_D_SXTW: - printf("sxtw"); - break; - - default: - assert(!"Bad value"); - break; - } -} - -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding along with the N value - */ - -void emitter::emitDispSveExtendOptsModN(insOpts opt, ssize_t imm) -{ - assert(imm >= 0 && imm <= 3); - - if (imm == 0 && opt != INS_OPTS_LSL) - { - emitDispSveExtendOpts(opt); - } - else if (imm > 0) - { - emitDispSveExtendOpts(opt); - printf(" #%d", (int)imm); - } -} - -/***************************************************************************** - * - * Prints the encoding for the or LSL encoding along with the N value - * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], - * [{, , LSL #N}] - */ -void emitter::emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts opt, insFormat fmt) -{ - printf("["); - - if (isVectorRegister(reg1)) - { - // If the overall instruction is working on 128-bit - // registers, the size of this register for - // the mod addr is always 64-bit. - // Example: LD1Q {.Q }, /Z, [.D{, }] - if (opt == INS_OPTS_SCALABLE_Q) - { - emitDispSveReg(reg1, INS_OPTS_SCALABLE_D, reg2 != REG_ZR); - } - else - { - emitDispSveReg(reg1, opt, reg2 != REG_ZR); - } - } - else - { - emitDispReg(reg1, EA_8BYTE, reg2 != REG_ZR); - } - - if (isVectorRegister(reg2)) - { - emitDispSveReg(reg2, opt, false); - } - else if (reg2 != REG_ZR) + if (!strictArmAsm && hasShift) { - emitDispReg(reg2, EA_8BYTE, false); + imm <<= shiftAmount; } - - if (insOptsScalable32bitExtends(opt)) + emitDispImm(imm, false); + if (strictArmAsm && hasShift) { - emitDispComma(); - emitDispSveExtendOptsModN(opt, insSveGetLslOrModN(ins, fmt)); + printf(", LSL #%u", shiftAmount); } - // Omit 'lsl #N' only if the second register is ZR. - else if ((reg2 != REG_ZR) && insSveIsLslN(ins, fmt)) - { - emitDispComma(); - switch (insSveGetLslOrModN(ins, fmt)) - { - case 4: - printf("lsl #4"); - break; - - case 3: - printf("lsl #3"); - break; - - case 2: - printf("lsl #2"); - break; +} - case 1: - printf("lsl #1"); - break; +/***************************************************************************** + * + * Display an ARM64 condition code for the conditional instructions + */ +void emitter::emitDispCond(insCond cond) +{ + const static char* armCond[16] = {"eq", "ne", "hs", "lo", "mi", "pl", "vs", "vc", + "hi", "ls", "ge", "lt", "gt", "le", "AL", "NV"}; // The last two are invalid + unsigned imm = (unsigned)cond; + assert((0 <= imm) && (imm < ArrLen(armCond))); + printf(armCond[imm]); +} - default: - assert(!"Invalid instruction"); - break; - } - } - printf("]"); +/***************************************************************************** + * + * Display an ARM64 flags for the conditional instructions + */ +void emitter::emitDispFlags(insCflags flags) +{ + const static char* armFlags[16] = {"0", "v", "c", "cv", "z", "zv", "zc", "zcv", + "n", "nv", "nc", "ncv", "nz", "nzv", "nzc", "nzcv"}; + unsigned imm = (unsigned)flags; + assert((0 <= imm) && (imm < ArrLen(armFlags))); + printf(armFlags[imm]); } /***************************************************************************** * - * Prints the encoding for format [.S{, #}] + * Display an ARM64 'barrier' for the memory barrier instructions */ -void emitter::emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt) +void emitter::emitDispBarrier(insBarrier barrier) { - printf("["); - emitDispSveReg(reg1, opt, imm != 0); - if (imm != 0) - { - // This does not have to be printed as hex. - // We only do it because the capstone disassembly displays this immediate as hex. - // We could not modify capstone without affecting other cases. - emitDispImm(imm, false, /* alwaysHex */ true); - } - printf("]"); + const static char* armBarriers[16] = {"#0", "oshld", "oshst", "osh", "#4", "nshld", "nshst", "nsh", + "#8", "ishld", "ishst", "ish", "#12", "ld", "st", "sy"}; + unsigned imm = (unsigned)barrier; + assert((0 <= imm) && (imm < ArrLen(armBarriers))); + printf(armBarriers[imm]); } /***************************************************************************** * - * Prints the encoding for format [{, #, MUL VL}] + * Prints the encoding for the Shift Type encoding */ -void emitter::emitDispSveImmMulVl(regNumber reg1, ssize_t imm) + +void emitter::emitDispShiftOpts(insOpts opt) { - printf("["); - emitDispReg(reg1, EA_8BYTE, imm != 0); - if (imm != 0) - { - emitDispImm(imm, true); - printf("mul vl"); - } - printf("]"); + if (opt == INS_OPTS_LSL) + printf(" LSL "); + else if (opt == INS_OPTS_LSR) + printf(" LSR "); + else if (opt == INS_OPTS_ASR) + printf(" ASR "); + else if (opt == INS_OPTS_ROR) + printf(" ROR "); + else if (opt == INS_OPTS_MSL) + printf(" MSL "); + else + assert(!"Bad value"); } /***************************************************************************** * - * Prints the encoding for format [.D{, #}] + * Prints the encoding for the Extend Type encoding */ -void emitter::emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm) + +void emitter::emitDispExtendOpts(insOpts opt) { - printf("["); - if (isVectorRegister(reg1)) - { - emitDispSveReg(reg1, opt, imm != 0); - } + if (opt == INS_OPTS_UXTB) + printf("UXTB"); + else if (opt == INS_OPTS_UXTH) + printf("UXTH"); + else if (opt == INS_OPTS_UXTW) + printf("UXTW"); + else if (opt == INS_OPTS_UXTX) + printf("UXTX"); + else if (opt == INS_OPTS_SXTB) + printf("SXTB"); + else if (opt == INS_OPTS_SXTH) + printf("SXTH"); + else if (opt == INS_OPTS_SXTW) + printf("SXTW"); + else if (opt == INS_OPTS_SXTX) + printf("SXTX"); else - { - emitDispReg(reg1, EA_8BYTE, imm != 0); - } - if (imm != 0) - { - // This does not have to be printed as hex. - // We only do it because the capstone disassembly displays this immediate as hex. - // We could not modify capstone without affecting other cases. - emitDispImm(imm, false, /* alwaysHex */ (imm > 31)); - } - printf("]"); + assert(!"Bad value"); } /***************************************************************************** @@ -21307,46 +16861,6 @@ void emitter::emitDispReg(regNumber reg, emitAttr attr, bool addComma) emitDispComma(); } -//------------------------------------------------------------------------ -// emitDispSveReg: Display a scalable vector register name -// -void emitter::emitDispSveReg(regNumber reg, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - - if (addComma) - emitDispComma(); -} - -//------------------------------------------------------------------------ -// emitDispSveReg: Display a scalable vector register name with an arrangement suffix -// -void emitter::emitDispSveReg(regNumber reg, insOpts opt, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - - if (opt != INS_OPTS_NONE) - { - assert(insOptsScalable(opt) || insOptsScalable32bitExtends(opt)); - emitDispArrangement(opt); - } - - if (addComma) - emitDispComma(); -} - -//------------------------------------------------------------------------ -// emitDispSveRegIndex: Display a scalable vector register with indexed element -// -void emitter::emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma) -{ - assert(isVectorRegister(reg)); - printf(emitSveRegName(reg)); - emitDispElementIndex(index, addComma); -} - //------------------------------------------------------------------------ // emitDispVectorReg: Display a SIMD vector register name with an arrangement suffix // @@ -21429,122 +16943,6 @@ void emitter::emitDispVectorElemList( } } -//------------------------------------------------------------------------ -// emitDispSveConsecutiveRegList: Display a SVE consecutive vector register list -// -void emitter::emitDispSveConsecutiveRegList(regNumber firstReg, unsigned listSize, insOpts opt, bool addComma) -{ - assert(isVectorRegister(firstReg)); - - regNumber currReg = firstReg; - - assert(listSize > 0); - - printf("{ "); - // We do not want the short-hand for list size of 1 or 2. - if ((listSize <= 2) || (((unsigned)currReg + listSize - 1) > (unsigned)REG_V31)) - { - for (unsigned i = 0; i < listSize; i++) - { - const bool notLastRegister = (i != listSize - 1); - emitDispSveReg(currReg, opt, notLastRegister); - currReg = (currReg == REG_V31) ? REG_V0 : REG_NEXT(currReg); - } - } - else - { - // short-hand. example: { z0.s - z2.s } which is the same as { z0.s, z1.s, z2.s } - emitDispSveReg(currReg, opt, false); - printf(" - "); - emitDispSveReg((regNumber)(currReg + listSize - 1), opt, false); - } - printf(" }"); - - if (addComma) - { - emitDispComma(); - } -} - -//------------------------------------------------------------------------ -// emitDispPredicateReg: Display a predicate register name with with an arrangement suffix -// -void emitter::emitDispPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) -{ - assert(isPredicateRegister(reg)); - printf(emitPredicateRegName(reg, ptype)); - - if (ptype == PREDICATE_MERGE) - { - printf("/m"); - } - else if (ptype == PREDICATE_ZERO) - { - printf("/z"); - } - else if (ptype == PREDICATE_SIZED || ptype == PREDICATE_N_SIZED) - { - emitDispElemsize(optGetSveElemsize(opt)); - } - - if (addComma) - emitDispComma(); -} - -//------------------------------------------------------------------------ -// emitDispPredicateRegPair: Display a pair of predicate registers -// -void emitter::emitDispPredicateRegPair(regNumber reg, insOpts opt) -{ - printf("{ "); - emitDispPredicateReg(reg, PREDICATE_SIZED, opt, true); - emitDispPredicateReg((regNumber)((unsigned)reg + 1), PREDICATE_SIZED, opt, false); - printf(" }, "); -} - -//------------------------------------------------------------------------ -// emitDispLowPredicateReg: Display a low predicate register name with with an arrangement suffix -// -void emitter::emitDispLowPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) -{ - assert(isLowPredicateRegister(reg)); - reg = (regNumber)((((unsigned)reg - REG_PREDICATE_FIRST) & 0x7) + REG_PREDICATE_FIRST); - emitDispPredicateReg(reg, ptype, opt, addComma); -} - -//------------------------------------------------------------------------ -// emitDispLowPredicateRegPair: Display a pair of low predicate registers -// -void emitter::emitDispLowPredicateRegPair(regNumber reg, insOpts opt) -{ - assert(isLowPredicateRegister(reg)); - - printf("{ "); - const unsigned baseRegNum = ((unsigned)reg - REG_PREDICATE_FIRST) & 0x7; - const unsigned regNum = (baseRegNum * 2) + REG_PREDICATE_FIRST; - emitDispPredicateReg((regNumber)regNum, PREDICATE_SIZED, opt, true); - emitDispPredicateReg((regNumber)(regNum + 1), PREDICATE_SIZED, opt, false); - printf(" }, "); -} - -//------------------------------------------------------------------------ -// emitDispVectorLengthSpecifier: Display the vector length specifier -// -void emitter::emitDispVectorLengthSpecifier(instrDesc* id) -{ - assert(id != nullptr); - assert(insOptsScalableStandard(id->idInsOpt())); - - if (id->idVectorLength4x()) - { - printf("vlx4"); - } - else - { - printf("vlx2"); - } -} - //------------------------------------------------------------------------ // emitDispArrangement: Display a SIMD vector arrangement suffix // @@ -21599,7 +16997,7 @@ void emitter::emitDispArrangement(insOpts opt) break; default: - assert(!"Invalid SVE insOpt"); + assert(!"Invalid insOpt"); } printf("."); printf(str); @@ -21814,103 +17212,6 @@ void emitter::emitDispAddrRRExt(regNumber reg1, regNumber reg2, insOpts opt, boo printf("]"); } -/***************************************************************************** - * - * Display an insSvePattern - */ -void emitter::emitDispSvePattern(insSvePattern pattern, bool addComma) -{ - printf("%s", svePatternNames[pattern]); - - if (addComma) - { - emitDispComma(); - } -} - -/***************************************************************************** - * - * Display an insSvePrfop - */ -void emitter::emitDispSvePrfop(insSvePrfop prfop, bool addComma) -{ - switch (prfop) - { - case SVE_PRFOP_PLDL1KEEP: - printf("pldl1keep"); - break; - - case SVE_PRFOP_PLDL1STRM: - printf("pldl1strm"); - break; - - case SVE_PRFOP_PLDL2KEEP: - printf("pldl2keep"); - break; - - case SVE_PRFOP_PLDL2STRM: - printf("pldl2strm"); - break; - - case SVE_PRFOP_PLDL3KEEP: - printf("pldl3keep"); - break; - - case SVE_PRFOP_PLDL3STRM: - printf("pldl3strm"); - break; - - case SVE_PRFOP_PSTL1KEEP: - printf("pstl1keep"); - break; - - case SVE_PRFOP_PSTL1STRM: - printf("pstl1strm"); - break; - - case SVE_PRFOP_PSTL2KEEP: - printf("pstl2keep"); - break; - - case SVE_PRFOP_PSTL2STRM: - printf("pstl2strm"); - break; - - case SVE_PRFOP_PSTL3KEEP: - printf("pstl3keep"); - break; - - case SVE_PRFOP_PSTL3STRM: - printf("pstl3strm"); - break; - - case SVE_PRFOP_CONST6: - printf("#6"); - break; - - case SVE_PRFOP_CONST7: - printf("#7"); - break; - - case SVE_PRFOP_CONST14: - printf("#0xE"); - break; - - case SVE_PRFOP_CONST15: - printf("#0xF"); - break; - - default: - assert(!"Invalid prfop"); - break; - } - - if (addComma) - { - emitDispComma(); - } -} - /***************************************************************************** * * Display (optionally) the instruction encoding in hex @@ -23381,7 +18682,7 @@ void emitter::emitDispInsHelp( { ssize_t imm1; ssize_t imm2; - insDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); emitDispSveReg(id->idReg1(), id->idInsOpt(), true); // ddddd emitDispImm(imm1, true); // iiiii emitDispImm(imm2, false); // iiiii diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 01a038c8a4dcc..08e0f14a766cb 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -741,14 +741,14 @@ static code_t insEncodeSveShift_23_to_22_9_to_0(emitAttr size, bool isRightShift // for an Arm64 Sve instruction. static code_t insEncodeSveElemsize_R_22(emitAttr size); -// Returns the immediate value for instructions that encode it as a difference from tszh:tszl:imm3. -static ssize_t insGetImmDiff(const ssize_t imm, const insOpts opt); +// Returns the immediate value for SVE instructions that encode it as a difference from tszh:tszl:imm3. +static ssize_t insSveGetImmDiff(const ssize_t imm, const insOpts opt); // Returns the two 5-bit signed immediates encoded as one ssize_t. -static ssize_t insEncodeTwoSimm5(ssize_t imm1, ssize_t imm2); +static ssize_t insSveEncodeTwoSimm5(ssize_t imm1, ssize_t imm2); -// Decodes imm into two 5-bit signed immediates, using the encoding format from insEncodeTwoSimm5. -static void insDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2); +// Decodes imm into two 5-bit signed immediates, using the encoding format from insSveEncodeTwoSimm5. +static void insSveDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2); // Returns the encoding to select an insSvePattern static code_t insEncodeSvePattern(insSvePattern pattern); diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 66f4cebd61cd8..d19528dd8f8cc 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -22,6 +22,23 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX #include "instr.h" +/*****************************************************************************/ + +// clang-format off +static const char * const svePatternNames[] = +{ + "pow2", "vl1", "vl2", "vl3", + "vl4", "vl5", "vl6", "vl7", + "vl8", "vl16", "vl32", "vl64", + "vl128", "vl256", "invalid", "invalid", + "invalid", "invalid", "invalid", "invalid", + "invalid", "invalid", "invalid", "invalid", + "invalid", "invalid", "invalid", "invalid", + "invalid", "mul4", "mul3", "all" +}; + +// clang-format on + /***************************************************************************** * * Add a SVE instruction with a single immediate value. @@ -832,7 +849,7 @@ void emitter::emitInsSve_R_I_I( assert(isValidSimm<5>(imm1)); // iiiii assert(isValidSimm<5>(imm2)); // iiiii assert(isValidVectorElemsize(optGetSveElemsize(opt))); // xx - immOut = insEncodeTwoSimm5(imm1, imm2); + immOut = insSveEncodeTwoSimm5(imm1, imm2); fmt = IF_SVE_AX_1A; } else @@ -5909,4 +5926,4691 @@ void emitter::emitIns_PRFOP_R_R_I(instruction ins, appendToCurIG(id); } +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0x00000000; + + case EA_2BYTE: + return 0x00400000; // set the bit at location 22 + + case EA_4BYTE: + return 0x00800000; // set the bit at location 23 + + case EA_8BYTE: + return 0x00C00000; // set the bit at location 23 and 22 + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the size at bit locations '22-21'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_22_to_21(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0; + + case EA_2BYTE: + return (1 << 21); // set the bit at location 21 + + case EA_4BYTE: + return (1 << 22); // set the bit at location 22 + + case EA_8BYTE: + return (1 << 22) | (1 << 21); // set the bit at location 22 and 21 + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the size at bit locations '18-17'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_18_to_17(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0; + + case EA_2BYTE: + return (1 << 17); // set the bit at location 17 + + case EA_4BYTE: + return (1 << 18); // set the bit at location 18 + + case EA_8BYTE: + return (1 << 18) | (1 << 17); // set the bit at location 18 and 17 + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'sz' at bit location '20'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_20(emitAttr size) +{ + switch (size) + { + case EA_4BYTE: + return 0; + + case EA_8BYTE: + return (1 << 20); + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'sz' at bit location '21'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_sz_21(emitAttr size) +{ + switch (size) + { + case EA_4BYTE: + return 0; + + case EA_8BYTE: + return (1 << 21); + + default: + assert(!"Invalid insOpt for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:20-19'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_23_tszl_20_to_19(emitAttr size) +{ + switch (size) + { + case EA_1BYTE: + return 0x080000; // set the bit at location 19 + + case EA_2BYTE: + return 0x100000; // set the bit at location 20 + + case EA_4BYTE: + return 0x400000; // set the bit at location 22 + + case EA_8BYTE: + return 0x800000; // set the bit at location 23 + + default: + assert(!"Invalid size for vector register"); + } + return 0; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8 byte elemsize for an Arm64 Sve vector instruction at bit location '30'. + * This only works on select formats. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_30_or_21(insFormat fmt, emitAttr size) +{ + switch (fmt) + { + case IF_SVE_HX_3A_B: + case IF_SVE_HX_3A_E: + switch (size) + { + case EA_4BYTE: + return 0; + + case EA_8BYTE: + return (1 << 30); + + default: + break; + } + + assert(!"Invalid size for vector register"); + return 0; + + case IF_SVE_IV_3A: + assert(size == EA_8BYTE); + return 0; + + case IF_SVE_JI_3A_A: + switch (size) + { + case EA_4BYTE: + return (1 << 21); + + case EA_8BYTE: + return 0; + + default: + break; + } + + assert(!"Invalid size for vector register"); + return 0; + + default: + break; + } + + assert(!"Unexpected instruction format"); + return 0; +} +/***************************************************************************** + * + * Returns the encoding for the field 'i1:tszh:tszl' at bit locations '23-22:20-18'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_tszh_tszl_and_imm(const insOpts opt, const ssize_t imm) +{ + code_t encoding = 0; + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<4>(imm)); + encoding = 0x040000; // set the bit at location 18 + // encode immediate at location 23-22:20-19 + encoding |= ((imm & 0b1100) << 22); + encoding |= ((imm & 0b11) << 19); + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); + encoding = 0x080000; // set the bit at location 19 + // encode immediate at location 23-22:20 + encoding |= ((imm & 0b110) << 22); + encoding |= ((imm & 1) << 20); + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + encoding = 0x100000; // set the bit at location 20 + encoding |= (imm << 22); // encode immediate at location 23:22 + break; + + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); + encoding = 0x400000; // set the bit at location 22 + encoding |= (imm << 23); // encode immediate at location 23 + break; + + default: + assert(!"Invalid size for vector register"); + break; + } + + return encoding; +} + +/***************************************************************************** + * + * Returns the encoding for the field 'tszh:tszl:imm3' at bit locations '23-22:20-19:18-16'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithShift_tszh_tszl_imm3(const insOpts opt, + ssize_t imm, + bool isRightShift) +{ + code_t encoding = 0; + + imm = insEncodeShiftImmediate(optGetSveElemsize(opt), isRightShift, imm); + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + imm = imm & 0b111; // bits 18-16 + encoding |= (1 << 19); // bit 19 + break; + + case INS_OPTS_SCALABLE_H: + imm = imm & 0b1111; // bits 19-16 + encoding |= (1 << 20); // bit 20 + break; + + case INS_OPTS_SCALABLE_S: + imm = imm & 0b11111; // bits 20-16 + encoding |= (1 << 22); // bit 22 + break; + + case INS_OPTS_SCALABLE_D: + // this gets the last bit of 'imm' and tries to set bit 22 + encoding |= ((imm >> 5) << 22); + imm = imm & 0b11111; // bits 20-16 + encoding |= (1 << 23); // bit 23 + break; + + default: + assert(!"Invalid size for vector register"); + break; + } + + return (encoding | (code_t)(imm << 16)); +} + +/***************************************************************************** + * + * Returns the encoding for the field 'i1:tsz' at bit locations '20:19-16'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsizeWithImmediate_i1_tsz(const insOpts opt, ssize_t imm) +{ + code_t encoding = 0; + + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimm<4>(imm)); + encoding |= (1 << 16); // bit 16 + encoding |= (imm << 17); // bits 20-17 + break; + + case INS_OPTS_SCALABLE_H: + assert(isValidUimm<3>(imm)); + encoding |= (1 << 17); // bit 17 + encoding |= (imm << 18); // bits 20-18 + break; + + case INS_OPTS_SCALABLE_S: + assert(isValidUimm<2>(imm)); + encoding |= (1 << 18); // bit 18 + encoding |= (imm << 19); // bits 20-19 + break; + + case INS_OPTS_SCALABLE_D: + assert(isValidUimm<1>(imm)); + encoding |= (1 << 19); // bit 19 + encoding |= (imm << 20); // bit 20 + break; + + default: + assert(!"Invalid size for vector register"); + break; + } + + return encoding; +} + +/***************************************************************************** + * + * Returns the encoding to select the elemsize for an Arm64 SVE vector instruction plus an immediate. + * This specifically encodes the field 'tszh:tszl' at bit locations '23-22:9-8'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveShift_23_to_22_9_to_0(emitAttr size, bool isRightShift, size_t imm) +{ + code_t encodedSize = 0; + + switch (size) + { + case EA_1BYTE: + encodedSize = 0x100; // set the bit at location 8 + break; + + case EA_2BYTE: + encodedSize = 0x200; // set the bit at location 9 + break; + + case EA_4BYTE: + encodedSize = 0x400000; // set the bit at location 22 + break; + + case EA_8BYTE: + encodedSize = 0x800000; // set the bit at location 23 + break; + + default: + assert(!"Invalid esize for vector register"); + } + + code_t encodedImm = insEncodeShiftImmediate(size, isRightShift, imm); + code_t imm3High = (encodedImm & 0x60) << 17; + code_t imm3Low = (encodedImm & 0x1f) << 5; + return encodedSize | imm3High | imm3Low; +} + +/***************************************************************************** + * + * Returns the encoding to select the constant values 90 or 270 for an Arm64 SVE vector instruction + * This specifically encode the field 'rot' at bit location '16'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveImm90_or_270_rot(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm90_or_270(imm)); + return (code_t)(imm << 16); +} + +/***************************************************************************** + * + * Returns the encoding to select the constant values 0, 90, 180 or 270 for an Arm64 SVE vector instruction + * This specifically encode the field 'rot' at bit locations '14-13'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveImm0_to_270_rot(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm0_to_270(imm)); + return (code_t)(imm << 13); +} + +/***************************************************************************** + * + * Returns the encoding to select the constant float values 0, 0.5, 1.0 or 2.0 for an Arm64 SVE vector instruction + * This specifically encode the field 'i1' at bit location '5'. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveSmallFloatImm(ssize_t imm) +{ + assert(emitIsValidEncodedSmallFloatImm(imm)); + return (code_t)(imm << 5); +} + +/***************************************************************************** + * + * Returns the register list size for the given SVE instruction. + */ + +/*static*/ int emitter::insGetSveReg1ListSize(instruction ins) +{ + switch (ins) + { + case INS_sve_ld1d: + case INS_sve_ld1w: + case INS_sve_ld1sw: + case INS_sve_ld1sb: + case INS_sve_ld1b: + case INS_sve_ld1sh: + case INS_sve_ld1h: + case INS_sve_ldnf1d: + case INS_sve_ldnf1sw: + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1b: + case INS_sve_ldnt1b: + case INS_sve_ldnt1d: + case INS_sve_ldnt1h: + case INS_sve_ldnt1w: + case INS_sve_ld1rob: + case INS_sve_ld1rod: + case INS_sve_ld1roh: + case INS_sve_ld1row: + case INS_sve_ld1rqb: + case INS_sve_ld1rqd: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + case INS_sve_stnt1b: + case INS_sve_stnt1d: + case INS_sve_stnt1h: + case INS_sve_stnt1w: + case INS_sve_st1d: + case INS_sve_st1w: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + case INS_sve_ldff1h: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + case INS_sve_st1b: + case INS_sve_st1h: + case INS_sve_ldff1sb: + case INS_sve_ldff1b: + case INS_sve_ldnt1sb: + case INS_sve_ldnt1sh: + case INS_sve_ld1rd: + case INS_sve_ld1rsw: + case INS_sve_ld1rh: + case INS_sve_ld1rsb: + case INS_sve_ld1rsh: + case INS_sve_ld1rw: + case INS_sve_ld1q: + case INS_sve_ldnt1sw: + case INS_sve_st1q: + case INS_sve_ld1rb: + return 1; + + case INS_sve_ld2b: + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + case INS_sve_ld2q: + case INS_sve_splice: // SVE_CV_3A + case INS_sve_st2b: + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: + case INS_sve_st2q: + case INS_sve_whilege: // SVE_DX_3A + case INS_sve_whilegt: // SVE_DX_3A + case INS_sve_whilehi: // SVE_DX_3A + case INS_sve_whilehs: // SVE_DX_3A + case INS_sve_whilele: // SVE_DX_3A + case INS_sve_whilels: // SVE_DX_3A + case INS_sve_whilelt: // SVE_DX_3A + case INS_sve_pext: // SVE_DW_2B + return 2; + + case INS_sve_ld3b: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + case INS_sve_ld3q: + case INS_sve_st3b: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: + case INS_sve_st3q: + return 3; + + case INS_sve_ld4b: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + case INS_sve_ld4q: + case INS_sve_st4b: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + case INS_sve_st4q: + return 4; + + default: + assert(!"Unexpected instruction"); + return 1; + } +} + +/***************************************************************************** + * + * Returns the predicate type for the given SVE format. + */ + +/*static*/ emitter::PredicateType emitter::insGetPredicateType(insFormat fmt, int regpos /* =0 */) +{ + switch (fmt) + { + case IF_SVE_BV_2A: + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + case IF_SVE_HW_4A_B: + case IF_SVE_HW_4A_C: + case IF_SVE_HW_4B: + case IF_SVE_HW_4B_D: + case IF_SVE_HX_3A_E: + case IF_SVE_IJ_3A_D: + case IF_SVE_IJ_3A_E: + case IF_SVE_IJ_3A_F: + case IF_SVE_IK_4A_G: + case IF_SVE_IJ_3A_G: + case IF_SVE_IK_4A_I: + case IF_SVE_IH_3A_F: + case IF_SVE_II_4A_H: + case IF_SVE_IH_3A: + case IF_SVE_IH_3A_A: + case IF_SVE_II_4A: + case IF_SVE_II_4A_B: + case IF_SVE_IU_4A: + case IF_SVE_IU_4A_C: + case IF_SVE_IU_4B: + case IF_SVE_IU_4B_D: + case IF_SVE_IV_3A: + case IF_SVE_IG_4A_F: + case IF_SVE_IG_4A_G: + case IF_SVE_IJ_3A: + case IF_SVE_IK_4A: + case IF_SVE_IK_4A_F: + case IF_SVE_IK_4A_H: + case IF_SVE_IU_4A_A: + case IF_SVE_IU_4B_B: + case IF_SVE_HX_3A_B: + case IF_SVE_IG_4A: + case IF_SVE_IG_4A_D: + case IF_SVE_IG_4A_E: + case IF_SVE_IF_4A: + case IF_SVE_IF_4A_A: + case IF_SVE_IM_3A: + case IF_SVE_IN_4A: + case IF_SVE_IX_4A: + case IF_SVE_IO_3A: + case IF_SVE_IP_4A: + case IF_SVE_IQ_3A: + case IF_SVE_IR_4A: + case IF_SVE_IS_3A: + case IF_SVE_IT_4A: + case IF_SVE_GI_4A: + case IF_SVE_IC_3A_C: + case IF_SVE_IC_3A: + case IF_SVE_IC_3A_B: + case IF_SVE_IC_3A_A: + case IF_SVE_IL_3A_C: + case IF_SVE_IL_3A: + case IF_SVE_IL_3A_B: + case IF_SVE_IL_3A_A: + case IF_SVE_IW_4A: + return PREDICATE_ZERO; + + case IF_SVE_BV_2A_J: + case IF_SVE_CP_3A: + case IF_SVE_CQ_3A: + case IF_SVE_AM_2A: + case IF_SVE_AN_3A: + case IF_SVE_AO_3A: + case IF_SVE_HL_3A: + case IF_SVE_HM_2A: + case IF_SVE_AA_3A: + case IF_SVE_BU_2A: + case IF_SVE_BV_2B: + case IF_SVE_HS_3A: + case IF_SVE_HP_3A: + case IF_SVE_HP_3B: + case IF_SVE_AR_4A: + case IF_SVE_BV_2A_A: + case IF_SVE_AB_3A: + case IF_SVE_ET_3A: + case IF_SVE_HU_4A: + case IF_SVE_HL_3B: + case IF_SVE_AD_3A: + case IF_SVE_AB_3B: + case IF_SVE_AE_3A: + case IF_SVE_EU_3A: + case IF_SVE_GT_4A: + case IF_SVE_AP_3A: + case IF_SVE_HO_3A: + case IF_SVE_HO_3B: + case IF_SVE_HO_3C: + case IF_SVE_GQ_3A: + case IF_SVE_HU_4B: + case IF_SVE_AQ_3A: + case IF_SVE_CU_3A: + case IF_SVE_AC_3A: + case IF_SVE_ER_3A: + case IF_SVE_GR_3A: + case IF_SVE_ES_3A: + case IF_SVE_HR_3A: + case IF_SVE_EP_3A: + case IF_SVE_GP_3A: + case IF_SVE_EQ_3A: + case IF_SVE_HQ_3A: + case IF_SVE_AS_4A: + case IF_SVE_CT_3A: + case IF_SVE_HV_4A: + return PREDICATE_MERGE; + + case IF_SVE_CZ_4A_A: + case IF_SVE_CZ_4A_L: + case IF_SVE_CE_2A: + case IF_SVE_CE_2B: + case IF_SVE_CE_2C: + case IF_SVE_CE_2D: + case IF_SVE_CF_2A: + case IF_SVE_CF_2B: + case IF_SVE_CF_2C: + case IF_SVE_CF_2D: + case IF_SVE_CI_3A: + case IF_SVE_CJ_2A: + case IF_SVE_DE_1A: + case IF_SVE_DH_1A: + case IF_SVE_DJ_1A: + case IF_SVE_DM_2A: + case IF_SVE_DN_2A: + case IF_SVE_DO_2A: + case IF_SVE_DP_2A: + case IF_SVE_DR_1A: + case IF_SVE_DT_3A: + case IF_SVE_DU_3A: + case IF_SVE_CK_2A: + return PREDICATE_SIZED; + + case IF_SVE_DB_3A: + // Second register could be ZERO or MERGE so handled at source. + assert(regpos != 2); + return PREDICATE_SIZED; + + case IF_SVE_DL_2A: + case IF_SVE_DY_3A: + case IF_SVE_DZ_1A: + return PREDICATE_N_SIZED; + + // This is a special case as the second register could be ZERO or MERGE. + // / + // Therefore, by default return NONE due to ambiguity. + case IF_SVE_AH_3A: + // TODO: Handle these cases. + assert(false); + break; + + case IF_SVE_JD_4B: + case IF_SVE_JD_4C: + case IF_SVE_JI_3A_A: + case IF_SVE_JJ_4A: + case IF_SVE_JJ_4A_B: + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + case IF_SVE_JJ_4B: + case IF_SVE_JJ_4B_E: + case IF_SVE_JN_3B: + case IF_SVE_JN_3C: + case IF_SVE_JD_4A: + case IF_SVE_JN_3A: + case IF_SVE_JD_4C_A: + case IF_SVE_JJ_4B_C: + case IF_SVE_JL_3A: + case IF_SVE_JN_3C_D: + case IF_SVE_HY_3A: + case IF_SVE_HY_3A_A: + case IF_SVE_HY_3B: + case IF_SVE_HZ_2A_B: + case IF_SVE_IA_2A: + case IF_SVE_IB_3A: + case IF_SVE_JK_4A: + case IF_SVE_JK_4A_B: + case IF_SVE_JK_4B: + case IF_SVE_IZ_4A: + case IF_SVE_IZ_4A_A: + case IF_SVE_JB_4A: + case IF_SVE_JM_3A: + case IF_SVE_CM_3A: + case IF_SVE_CN_3A: + case IF_SVE_CO_3A: + case IF_SVE_JA_4A: + case IF_SVE_CR_3A: + case IF_SVE_CS_3A: + case IF_SVE_CV_3A: + case IF_SVE_CV_3B: + case IF_SVE_DW_2A: // [] + case IF_SVE_DW_2B: // [] + case IF_SVE_JC_4A: + case IF_SVE_JO_3A: + case IF_SVE_JE_3A: + case IF_SVE_JF_4A: + case IF_SVE_AK_3A: + case IF_SVE_HE_3A: + case IF_SVE_AF_3A: + case IF_SVE_AG_3A: + case IF_SVE_AI_3A: + case IF_SVE_AJ_3A: + case IF_SVE_AL_3A: + case IF_SVE_CL_3A: + case IF_SVE_GS_3A: + case IF_SVE_HJ_3A: + case IF_SVE_IY_4A: + return PREDICATE_NONE; + + case IF_SVE_CX_4A: + case IF_SVE_CX_4A_A: + case IF_SVE_CY_3A: + case IF_SVE_CY_3B: + case IF_SVE_GE_4A: + case IF_SVE_HT_4A: + assert((regpos == 1) || (regpos == 2)); + return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_CZ_4A: + case IF_SVE_DA_4A: + case IF_SVE_DB_3B: + case IF_SVE_DC_3A: + assert((regpos >= 1) && (regpos <= 4)); + return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_CZ_4A_K: + assert((regpos >= 1) && (regpos <= 3)); + return (regpos == 2 ? PREDICATE_MERGE : PREDICATE_SIZED); + + case IF_SVE_DD_2A: + case IF_SVE_DF_2A: + assert((regpos >= 1) && (regpos <= 3)); + return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_DG_2A: + return (regpos == 2 ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_DI_2A: + return (regpos == 1 ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_DK_3A: + assert((regpos == 2) || (regpos == 3)); + return ((regpos == 2) ? PREDICATE_NONE : PREDICATE_SIZED); + + case IF_SVE_HI_3A: + assert((regpos == 1) || (regpos == 2)); + return ((regpos == 2) ? PREDICATE_ZERO : PREDICATE_SIZED); + + case IF_SVE_DV_4A: + assert((regpos >= 1) && (regpos <= 3)); + return ((regpos == 3) ? PREDICATE_SIZED : PREDICATE_NONE); + + case IF_SVE_ID_2A: + case IF_SVE_JG_2A: + return PREDICATE_NONE; + + default: + break; + } + + assert(!"Unexpected instruction format"); + return PREDICATE_NONE; +} + +/***************************************************************************** + * + * Returns true if the SVE instruction has a LSL addr. + * This is for formats that have [, , LSL #N], [{, , LSL #N}] + */ +/*static*/ bool emitter::insSveIsLslN(instruction ins, insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_JD_4A: + switch (ins) + { + case INS_sve_st1h: + return true; + + default: + break; + } + break; + + case IF_SVE_JD_4B: + switch (ins) + { + case INS_sve_st1w: + return true; + + default: + break; + } + break; + + case IF_SVE_HW_4B: + switch (ins) + { + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ld1w: + case INS_sve_ldff1w: + return true; + + default: + break; + } + break; + + case IF_SVE_IG_4A: + switch (ins) + { + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + return true; + + default: + break; + } + break; + + case IF_SVE_IG_4A_F: + switch (ins) + { + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; + + default: + break; + } + break; + + case IF_SVE_IG_4A_G: + switch (ins) + { + case INS_sve_ldff1h: + return true; + + default: + break; + } + break; + + case IF_SVE_II_4A: + case IF_SVE_II_4A_B: + switch (ins) + { + case INS_sve_ld1d: + return true; + + default: + break; + } + break; + + case IF_SVE_II_4A_H: + switch (ins) + { + case INS_sve_ld1w: + return true; + + default: + break; + } + break; + + case IF_SVE_IK_4A: + switch (ins) + { + case INS_sve_ld1sw: + return true; + + default: + break; + } + break; + + case IF_SVE_IK_4A_G: + switch (ins) + { + case INS_sve_ld1sh: + return true; + + default: + break; + } + break; + + case IF_SVE_IK_4A_I: + switch (ins) + { + case INS_sve_ld1h: + return true; + + default: + break; + } + break; + + case IF_SVE_IN_4A: + switch (ins) + { + case INS_sve_ldnt1d: + case INS_sve_ldnt1h: + case INS_sve_ldnt1w: + return true; + + default: + break; + } + break; + + case IF_SVE_IP_4A: + switch (ins) + { + case INS_sve_ld1roh: + case INS_sve_ld1row: + case INS_sve_ld1rod: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + case INS_sve_ld1rqd: + return true; + + default: + break; + } + break; + + case IF_SVE_IR_4A: + switch (ins) + { + case INS_sve_ld2q: + case INS_sve_ld3q: + case INS_sve_ld4q: + return true; + + default: + break; + } + break; + + case IF_SVE_IT_4A: + switch (ins) + { + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + return true; + + default: + break; + } + break; + + case IF_SVE_IU_4B: + switch (ins) + { + case INS_sve_ld1sw: + case INS_sve_ldff1sw: + case INS_sve_ld1d: + case INS_sve_ldff1d: + return true; + + default: + break; + } + break; + + case IF_SVE_JB_4A: + switch (ins) + { + case INS_sve_stnt1h: + case INS_sve_stnt1w: + case INS_sve_stnt1d: + return true; + + default: + break; + } + break; + + case IF_SVE_JC_4A: + switch (ins) + { + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + return true; + + default: + break; + } + break; + + case IF_SVE_JD_4C: + switch (ins) + { + case INS_sve_st1w: + case INS_sve_st1d: + return true; + + default: + break; + } + break; + + case IF_SVE_JD_4C_A: + switch (ins) + { + case INS_sve_st1d: + return true; + + default: + break; + } + break; + + case IF_SVE_JF_4A: + switch (ins) + { + case INS_sve_st2q: + case INS_sve_st3q: + case INS_sve_st4q: + return true; + + default: + break; + } + break; + + case IF_SVE_JJ_4B: + switch (ins) + { + case INS_sve_st1h: + case INS_sve_st1w: + case INS_sve_st1d: + return true; + + default: + break; + } + break; + + case IF_SVE_HY_3B: + case IF_SVE_IB_3A: + switch (ins) + { + case INS_sve_prfh: + case INS_sve_prfw: + case INS_sve_prfd: + return true; + + default: + break; + } + break; + + default: + break; + } + + return false; +} + +/***************************************************************************** + * + * Returns true if the SVE instruction has a addr. + * This is for formats that have [, .T, ], [, .T, #N] + */ +/*static*/ bool emitter::insSveIsModN(instruction ins, insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_JJ_4A: + case IF_SVE_JJ_4A_B: + switch (ins) + { + case INS_sve_st1d: + case INS_sve_st1h: + case INS_sve_st1w: + return true; + + default: + break; + } + break; + + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + switch (ins) + { + case INS_sve_st1h: + case INS_sve_st1w: + return true; + + default: + break; + } + break; + + case IF_SVE_JK_4A: + case IF_SVE_JK_4A_B: + switch (ins) + { + case INS_sve_st1b: + return true; + + default: + break; + } + break; + + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + switch (ins) + { + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ld1w: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + case INS_sve_ldff1sb: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; + + default: + break; + } + break; + + case IF_SVE_HW_4A_B: + case IF_SVE_HW_4A_C: + switch (ins) + { + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ld1w: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; + + default: + break; + } + break; + + case IF_SVE_IU_4A: + switch (ins) + { + case INS_sve_ld1d: + case INS_sve_ld1sw: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + return true; + + default: + break; + } + break; + + case IF_SVE_IU_4A_A: + switch (ins) + { + case INS_sve_ld1sw: + case INS_sve_ldff1d: + case INS_sve_ldff1sw: + return true; + + default: + break; + } + break; + + case IF_SVE_IU_4A_C: + switch (ins) + { + case INS_sve_ld1d: + return true; + + default: + break; + } + break; + + case IF_SVE_HY_3A: + case IF_SVE_HY_3A_A: + switch (ins) + { + case INS_sve_prfb: + case INS_sve_prfh: + case INS_sve_prfw: + case INS_sve_prfd: + return true; + + default: + break; + } + break; + + default: + break; + } + + return false; +} + +/***************************************************************************** + * + * Returns 0, 1, 2, 3 or 4 depending on the instruction and format. + * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], + * [{, , LSL #N}] + */ + +/*static*/ int emitter::insSveGetLslOrModN(instruction ins, insFormat fmt) +{ + switch (fmt) + { + case IF_SVE_JD_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st1h: + return 1; + + default: + break; + } + break; + + case IF_SVE_JD_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st1w: + return 2; + + default: + break; + } + break; + + case IF_SVE_HW_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + return 1; + + case INS_sve_ld1w: + case INS_sve_ldff1w: + return 2; + + default: + break; + } + break; + + case IF_SVE_JJ_4A: + case IF_SVE_JJ_4A_B: + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + case IF_SVE_JK_4A: + case IF_SVE_JK_4A_B: + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + case IF_SVE_HW_4A_B: + case IF_SVE_HW_4A_C: + case IF_SVE_IU_4A: + case IF_SVE_IU_4A_A: + case IF_SVE_IU_4A_C: + assert(!insSveIsLslN(ins, fmt)); + assert(insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1h: + case INS_sve_ld1sh: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + switch (fmt) + { + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + return 1; + + default: + break; + } + return 0; + + case INS_sve_ld1w: + case INS_sve_ldff1w: + case INS_sve_ld1sw: + case INS_sve_ldff1sw: + switch (fmt) + { + case IF_SVE_HW_4A: + case IF_SVE_HW_4A_A: + case IF_SVE_IU_4A: + return 2; + + default: + break; + } + return 0; + + case INS_sve_ld1d: + case INS_sve_ldff1d: + switch (fmt) + { + case IF_SVE_IU_4A: + return 3; + + default: + break; + } + return 0; + + case INS_sve_st1h: + switch (fmt) + { + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + return 0; + + default: + break; + } + return 1; + + case INS_sve_st1w: + switch (fmt) + { + case IF_SVE_JJ_4A_C: + case IF_SVE_JJ_4A_D: + return 0; + + default: + break; + } + return 2; + + case INS_sve_st1d: + if (fmt == IF_SVE_JJ_4A_B) + { + return 0; + } + return 3; + + default: + break; + } + return 0; + + case IF_SVE_IG_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldff1sw: + return 2; + + case INS_sve_ldff1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_IG_4A_F: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldff1sh: + return 1; + + case INS_sve_ldff1w: + return 2; + + default: + break; + } + break; + + case IF_SVE_IG_4A_G: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldff1h: + return 1; + + default: + break; + } + break; + + case IF_SVE_II_4A: + case IF_SVE_II_4A_B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_II_4A_H: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1w: + return 2; + + default: + break; + } + break; + + case IF_SVE_IK_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1sw: + return 2; + + default: + break; + } + break; + + case IF_SVE_IK_4A_G: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1sh: + return 1; + + default: + break; + } + break; + + case IF_SVE_IK_4A_I: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1h: + return 1; + + default: + break; + } + break; + + case IF_SVE_IN_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ldnt1h: + return 1; + case INS_sve_ldnt1w: + return 2; + case INS_sve_ldnt1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_IP_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1roh: + case INS_sve_ld1rqh: + return 1; + + case INS_sve_ld1row: + case INS_sve_ld1rqw: + return 2; + case INS_sve_ld1rod: + case INS_sve_ld1rqd: + return 3; + + default: + break; + } + break; + + case IF_SVE_IR_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld2q: + case INS_sve_ld3q: + case INS_sve_ld4q: + return 4; + + default: + break; + } + break; + + case IF_SVE_IT_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld2h: + case INS_sve_ld3h: + case INS_sve_ld4h: + return 1; + + case INS_sve_ld2w: + case INS_sve_ld3w: + case INS_sve_ld4w: + return 2; + + case INS_sve_ld2d: + case INS_sve_ld3d: + case INS_sve_ld4d: + return 3; + + default: + break; + } + break; + + case IF_SVE_IU_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_ld1sw: + case INS_sve_ldff1sw: + return 2; + + case INS_sve_ld1d: + case INS_sve_ldff1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_JB_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_stnt1h: + return 1; + + case INS_sve_stnt1w: + return 2; + + case INS_sve_stnt1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_JC_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st2h: + case INS_sve_st3h: + case INS_sve_st4h: + return 1; + + case INS_sve_st2w: + case INS_sve_st3w: + case INS_sve_st4w: + return 2; + + case INS_sve_st2d: + case INS_sve_st3d: + case INS_sve_st4d: + return 3; + + default: + break; + } + break; + + case IF_SVE_JD_4C: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st1w: + return 2; + + case INS_sve_st1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_JD_4C_A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_JF_4A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st2q: + case INS_sve_st3q: + case INS_sve_st4q: + return 4; + + default: + break; + } + break; + + case IF_SVE_JJ_4B: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_st1h: + return 1; + + case INS_sve_st1w: + return 2; + + case INS_sve_st1d: + return 3; + + default: + break; + } + break; + + case IF_SVE_HY_3A: + case IF_SVE_HY_3A_A: + assert(!insSveIsLslN(ins, fmt)); + assert(insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_prfb: + return 0; + + case INS_sve_prfh: + return 1; + + case INS_sve_prfw: + return 2; + + case INS_sve_prfd: + return 3; + + default: + break; + } + break; + + case IF_SVE_HY_3B: + case IF_SVE_IB_3A: + assert(insSveIsLslN(ins, fmt)); + assert(!insSveIsModN(ins, fmt)); + switch (ins) + { + case INS_sve_prfh: + return 1; + + case INS_sve_prfw: + return 2; + + case INS_sve_prfd: + return 3; + + default: + break; + } + break; + + default: + break; + } + + assert(!"Unexpected instruction format"); + return 0; +} + +/***************************************************************************** + * + * Returns true if the specified instruction can encode the 'dtype' field. + */ + +/*static*/ bool emitter::canEncodeSveElemsize_dtype(instruction ins) +{ + switch (ins) + { + case INS_sve_ld1w: + case INS_sve_ld1sb: + case INS_sve_ld1b: + case INS_sve_ld1sh: + case INS_sve_ld1h: + case INS_sve_ldnf1sh: + case INS_sve_ldnf1w: + case INS_sve_ldnf1h: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1b: + case INS_sve_ldff1b: + case INS_sve_ldff1sb: + case INS_sve_ldff1h: + case INS_sve_ldff1sh: + case INS_sve_ldff1w: + return true; + + default: + return false; + } +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * for the 'dtype' field. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype(instruction ins, emitAttr size, code_t code) +{ + assert(canEncodeSveElemsize_dtype(ins)); + assert(ins != INS_sve_ld1w); + switch (size) + { + case EA_1BYTE: + switch (ins) + { + case INS_sve_ld1b: + case INS_sve_ldnf1b: + case INS_sve_ldff1b: + return code; // By default, the instruction already encodes 8-bit. + + default: + assert(!"Invalid instruction for encoding dtype."); + } + return code; + + case EA_2BYTE: + switch (ins) + { + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1b: + case INS_sve_ldnf1h: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + return code | (1 << 21); // Set bit '21' to 1. + + case INS_sve_ld1sb: + case INS_sve_ldnf1sb: + case INS_sve_ldff1sb: + return code | (1 << 22); // Set bit '22' to 1. + + default: + assert(!"Invalid instruction for encoding dtype."); + } + return code; + + case EA_4BYTE: + switch (ins) + { + case INS_sve_ldnf1w: + case INS_sve_ldff1w: + return code; // By default, the instruction already encodes 32-bit. + + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1b: + case INS_sve_ldnf1h: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + return code | (1 << 22); // Set bit '22' to 1. + + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1sh: + case INS_sve_ldff1sb: + case INS_sve_ldff1sh: + return code | (1 << 21); // Set bit '21' to 1. + + default: + assert(!"Invalid instruction for encoding dtype."); + } + return code; + + case EA_8BYTE: + switch (ins) + { + case INS_sve_ldnf1w: + case INS_sve_ldff1w: + return code | (1 << 21); // Set bit '21' to 1. Set bit '15' to 1. + + case INS_sve_ld1b: + case INS_sve_ld1h: + case INS_sve_ldnf1b: + case INS_sve_ldnf1h: + case INS_sve_ldff1b: + case INS_sve_ldff1h: + return (code | (1 << 22)) | (1 << 21); // Set bit '22' and '21' to 1. + + case INS_sve_ld1sb: + case INS_sve_ld1sh: + case INS_sve_ldnf1sb: + case INS_sve_ldnf1sh: + case INS_sve_ldff1sb: + case INS_sve_ldff1sh: + return code; // By default, the instruction already encodes 64-bit. + + default: + assert(!"Invalid instruction for encoding dtype."); + } + return code; + + default: + assert(!"Invalid size for encoding dtype."); + } + + return code; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8/16 byte elemsize for the Arm64 Sve vector instruction 'ld1w' + * for the 'dtype' field. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtype_ld1w(instruction ins, + insFormat fmt, + emitAttr size, + code_t code) +{ + assert(canEncodeSveElemsize_dtype(ins)); + assert(ins == INS_sve_ld1w); + switch (size) + { + case EA_4BYTE: + switch (fmt) + { + case IF_SVE_IH_3A_F: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for S. + return (code | (1 << 15)) | (1 << 22); // Set bit '22' and '15' to 1. + + case IF_SVE_II_4A_H: + // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for S. + return (code | (1 << 14)) | (1 << 22); // Set bit '22' and '14' to 1. + + default: + break; + } + break; + + case EA_8BYTE: + switch (fmt) + { + case IF_SVE_IH_3A_F: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for D. + return ((code | (1 << 15)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '15' to 1. + + case IF_SVE_II_4A_H: + // Note: Bit '14' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for D. + return ((code | (1 << 14)) | (1 << 22)) | (1 << 21); // Set bit '22', '21' and '14' to 1. + + default: + break; + } + break; + + case EA_16BYTE: + switch (fmt) + { + case IF_SVE_IH_3A_F: + return code | (1 << 20); // Set bit '20' to 1. + + case IF_SVE_II_4A_H: + // Note: Bit '15' is not actually part of 'dtype', but it is necessary to set to '1' to get the + // proper encoding for Q. + return code | (1 << 15); // Set bit '15' to 1. + + default: + break; + } + break; + + default: + assert(!"Invalid size for encoding dtype."); + break; + } + + assert(!"Invalid instruction format"); + return code; +} + +/***************************************************************************** + * + * Returns the encoding to select the 1/2/4/8 byte elemsize for an Arm64 Sve vector instruction + * for the 'dtypeh' and 'dtypel' fields. + */ + +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_dtypeh_dtypel(instruction ins, + insFormat fmt, + emitAttr size, + code_t code) +{ + switch (fmt) + { + case IF_SVE_IC_3A_A: + switch (size) + { + case EA_4BYTE: + switch (ins) + { + case INS_sve_ld1rsh: + return code | (1 << 13); // set bit '13' + + case INS_sve_ld1rw: + return code | (1 << 14); // set bit '14' + + default: + break; + } + break; + + case EA_8BYTE: + switch (ins) + { + case INS_sve_ld1rsh: + return code; + + case INS_sve_ld1rw: + return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + + default: + break; + } + break; + + default: + break; + } + break; + + case IF_SVE_IC_3A_B: + switch (size) + { + case EA_2BYTE: + switch (ins) + { + case INS_sve_ld1rh: + return code | (1 << 13); // set bit '13' + + case INS_sve_ld1rsb: + return code | (1 << 24) | (1 << 14); // set bit '24' and '14' + + default: + break; + } + break; + + case EA_4BYTE: + switch (ins) + { + case INS_sve_ld1rh: + return code | (1 << 14); // set bit '14' + + case INS_sve_ld1rsb: + return code | (1 << 24) | (1 << 13); // set bit '24' and '13' + + default: + break; + } + break; + + case EA_8BYTE: + switch (ins) + { + case INS_sve_ld1rh: + return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + + case INS_sve_ld1rsb: + return code | (1 << 24); // set bit '24' + + default: + break; + } + break; + + default: + break; + } + break; + + case IF_SVE_IC_3A_C: + assert(ins == INS_sve_ld1rb); + switch (size) + { + case EA_1BYTE: + return code; + + case EA_2BYTE: + return code | (1 << 13); // set bit '13' + + case EA_4BYTE: + return code | (1 << 14); // set bit '14' + + case EA_8BYTE: + return code | (1 << 14) | (1 << 13); // set bits '14' and '13' + + default: + break; + } + break; + + default: + break; + } + + assert(!"Unexpected instruction format"); + return code; +} + +/***************************************************************************** + * + * Returns the encoding to select the 4/8-byte width specifier + * at bit location 22 for an Arm64 Sve instruction. + */ +/*static*/ emitter::code_t emitter::insEncodeSveElemsize_R_22(emitAttr size) +{ + if (size == EA_8BYTE) + { + return 0x400000; // set the bit at location 22 + } + + assert(size == EA_4BYTE); + return 0; +} + +/***************************************************************************** + * + * Returns the immediate value for SVE instructions that encode it as a difference + * from tszh:tszl:imm3. + */ +/*static*/ ssize_t emitter::insSveGetImmDiff(const ssize_t imm, const insOpts opt) +{ + switch (opt) + { + case INS_OPTS_SCALABLE_B: + assert(isValidUimmFrom1<3>(imm)); + return (8 - imm); + + case INS_OPTS_SCALABLE_H: + assert(isValidUimmFrom1<4>(imm)); + return (16 - imm); + + case INS_OPTS_SCALABLE_S: + assert(isValidUimmFrom1<5>(imm)); + return (32 - imm); + + case INS_OPTS_SCALABLE_D: + assert(isValidUimmFrom1<6>(imm)); + return (64 - imm); + + default: + unreached(); + break; + } + + return 0; +} + +/***************************************************************************** + * + * Returns the two 5-bit signed immediates encoded in the following format: + * njjj jjmi iiii + * - iiiii: the absolute value of imm1 + * - m: 1 if imm1 is negative, 0 otherwise + * - jjjjj: the absolute value of imm2 + * - n: 1 if imm2 is negative, 0 otherwise + */ +/*static*/ ssize_t emitter::insSveEncodeTwoSimm5(ssize_t imm1, ssize_t imm2) +{ + assert(isValidSimm<5>(imm1)); + assert(isValidSimm<5>(imm2)); + ssize_t immOut = 0; + + if (imm1 < 0) + { + // Set bit location 5 to indicate imm1 is negative + immOut |= 0x20; + imm1 *= -1; + } + + if (imm2 < 0) + { + // Set bit location 11 to indicate imm2 is negative + immOut |= 0x800; + imm2 *= -1; + } + + immOut |= imm1; + immOut |= (imm2 << 6); + return immOut; +} + +/***************************************************************************** + * + * Decodes imm into two 5-bit signed immediates, + * using the encoding format from insSveEncodeTwoSimm5. + */ +/*static*/ void emitter::insSveDecodeTwoSimm5(ssize_t imm, /* OUT */ ssize_t* const imm1, /* OUT */ ssize_t* const imm2) +{ + assert(imm1 != nullptr); + assert(imm2 != nullptr); + + *imm1 = (imm & 0x1F); + + if ((imm & 0x20) != 0) + { + *imm1 *= -1; + } + + imm >>= 6; + *imm2 = (imm & 0x1F); + + if ((imm & 0x20) != 0) + { + *imm2 *= -1; + } + + assert(isValidSimm<5>(*imm1)); + assert(isValidSimm<5>(*imm2)); +} + +/***************************************************************************** + * + * Returns the encoding to select an insSvePattern + */ +/*static*/ emitter::code_t emitter::insEncodeSvePattern(insSvePattern pattern) +{ + return (code_t)((unsigned)pattern << 5); +} + +/***************************************************************************** + * + * Returns the encoding for an immediate in the SVE variant of dup (indexed) + */ +/*static*/ emitter::code_t emitter::insEncodeSveBroadcastIndex(emitAttr elemsize, ssize_t index) +{ + unsigned lane_bytes = genLog2(elemsize) + 1; + code_t tsz = (1 << (lane_bytes - 1)); + code_t imm = (code_t)index << lane_bytes | tsz; + return insEncodeSplitUimm<23, 22, 20, 16>(imm); +} + +/***************************************************************************** + * + * Append the machine code corresponding to the given SVE instruction descriptor. + */ +BYTE* emitter::emitOutput_InstrSve(BYTE* dst, instrDesc* id) +{ + code_t code = 0; + instruction ins = id->idIns(); + insFormat fmt = id->idInsFmt(); + emitAttr size = id->idOpSize(); + + ssize_t imm; + + switch (fmt) + { + // Scalable. + case IF_SVE_AA_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise logical operations (predicated) + case IF_SVE_AB_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_AC_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer divide vectors (predicated) + case IF_SVE_AD_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer min/max/difference (predicated) + case IF_SVE_AE_3A: // ........xx...... ...gggmmmmmddddd -- SVE integer multiply vectors (predicated) + case IF_SVE_AF_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (predicated) + case IF_SVE_AG_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise logical reduction (quadwords) + case IF_SVE_AI_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (predicated) + case IF_SVE_AJ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (quadwords) + case IF_SVE_AK_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (predicated) + case IF_SVE_AL_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer min/max reduction (quadwords) + case IF_SVE_AN_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by vector (predicated) + case IF_SVE_AO_3A: // ........xx...... ...gggmmmmmddddd -- SVE bitwise shift by wide elements (predicated) + case IF_SVE_AP_3A: // ........xx...... ...gggnnnnnddddd -- SVE bitwise unary operations (predicated) + case IF_SVE_AQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer unary operations (predicated) + case IF_SVE_CL_3A: // ........xx...... ...gggnnnnnddddd -- SVE compress active elements + case IF_SVE_CM_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally broadcast element to vector + case IF_SVE_CN_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to SIMD&FP scalar + case IF_SVE_CP_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy SIMD&FP scalar register to vector + // (predicated) + case IF_SVE_CR_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to SIMD&FP scalar register + case IF_SVE_CU_3A: // ........xx...... ...gggnnnnnddddd -- SVE reverse within elements + case IF_SVE_EP_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer halving add/subtract (predicated) + case IF_SVE_EQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer pairwise add and accumulate long + case IF_SVE_ER_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 integer pairwise arithmetic + case IF_SVE_ES_3A: // ........xx...... ...gggnnnnnddddd -- SVE2 integer unary operations (predicated) + case IF_SVE_ET_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating add/subtract + case IF_SVE_EU_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 saturating/rounding bitwise shift left + // (predicated) + case IF_SVE_GR_3A: // ........xx...... ...gggmmmmmddddd -- SVE2 floating-point pairwise operations + case IF_SVE_GS_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction (quadwords) + case IF_SVE_HE_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point recursive reduction + case IF_SVE_HJ_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point serial reduction (predicated) + case IF_SVE_HL_3A: // ........xx...... ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + case IF_SVE_HQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point round to integral value + case IF_SVE_HR_3A: // ........xx...... ...gggnnnnnddddd -- SVE floating-point unary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm or nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AB_3B: // ................ ...gggmmmmmddddd -- SVE integer add/subtract vectors (predicated) + case IF_SVE_HL_3B: // ................ ...gggmmmmmddddd -- SVE floating-point arithmetic (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + // Scalable with Merge or Zero predicate + case IF_SVE_AH_3A: // ........xx.....M ...gggnnnnnddddd -- SVE constructive prefix (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // nnnnn + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // ddddd + code |= insEncodePredQualifier_16(id->idPredicateReg2Merge()); // M + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable with shift immediate + case IF_SVE_AM_2A: // ........xx...... ...gggxxiiiddddd -- SVE bitwise shift by immediate (predicated) + { + bool isRightShift = emitInsIsVectorRightShift(ins); + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= + insEncodeSveShift_23_to_22_9_to_0(optGetSveElemsize(id->idInsOpt()), isRightShift, imm); // xx, xxiii + dst += emitOutput_Instr(dst, code); + } + break; + + // Scalable, 4 regs. Reg4 in mmmmm. + case IF_SVE_AR_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE integer multiply-accumulate writing addend + // (predicated) + case IF_SVE_GI_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE2 histogram generation (vector) + case IF_SVE_HU_4A: // ........xx.mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable, 4 regs. Reg4 in aaaaa. + case IF_SVE_AS_4A: // ........xx.mmmmm ...gggaaaaaddddd -- SVE integer multiply-add writing multiplicand + // (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeReg_V_9_to_5(id->idReg4()); // aaaaa + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable, 3 regs, no predicates + case IF_SVE_AT_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_BD_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_BE_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 signed saturating doubling multiply high + // (unpredicated) + case IF_SVE_BG_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE bitwise shift by wide elements (unpredicated) + case IF_SVE_BK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point trig select coefficient + case IF_SVE_BR_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector segments + case IF_SVE_BZ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) + case IF_SVE_BZ_3A_A: // ........xx.mmmmm ......nnnnnddddd -- SVE table lookup (three sources) + case IF_SVE_CA_3A: // ........xx.mmmmm ......nnnnnddddd -- sve_int_perm_tbxquads + case IF_SVE_EH_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer dot product (unpredicated) + case IF_SVE_EL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply-add long + case IF_SVE_EM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add high + case IF_SVE_EN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add interleaved long + case IF_SVE_EO_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 saturating multiply-add long + case IF_SVE_EV_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE integer clamp + case IF_SVE_EX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE permute vector elements (quadwords) + case IF_SVE_FL_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long + case IF_SVE_FM_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract wide + case IF_SVE_FN_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_FP_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise exclusive-or interleaved + case IF_SVE_FQ_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 bitwise permute + case IF_SVE_FS_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract interleaved long + case IF_SVE_FW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate + case IF_SVE_FX_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer absolute difference and accumulate long + case IF_SVE_GC_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract narrow high part + case IF_SVE_GF_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE2 histogram generation (segment) + case IF_SVE_GW_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HK_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable, 3 regs, no predicates. General purpose source registers + case IF_SVE_BA_3A: // ........xx.mmmmm ......nnnnnddddd -- SVE index generation (register start, register + // increment) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeReg_Rm(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BH_3A: // .........x.mmmmm ....hhnnnnnddddd -- SVE address generation + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh + code |= insEncodeUimm<22, 22>(id->idInsOpt() == INS_OPTS_SCALABLE_D ? 1 : 0); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BH_3B: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + case IF_SVE_BH_3B_A: // ...........mmmmm ....hhnnnnnddddd -- SVE address generation + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // hh + dst += emitOutput_Instr(dst, code); + break; + + // Immediate and pattern to general purpose. + case IF_SVE_BL_1A: // ............iiii ......pppppddddd -- SVE element count + case IF_SVE_BM_1A: // ............iiii ......pppppddddd -- SVE inc/dec register by element count + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeUimm4From1_19_to_16(imm); // iiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BO_1A: // ...........Xiiii ......pppppddddd -- SVE saturating inc/dec register by element count + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeUimm4From1_19_to_16(imm); // iiii + code |= insEncodeSveElemsize_sz_20(id->idOpSize()); // X + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BQ_2A: // ...........iiiii ...iiinnnnnddddd -- SVE extract vector (immediate offset, destructive) + case IF_SVE_BQ_2B: // ...........iiiii ...iiimmmmmddddd -- SVE extract vector (immediate offset, destructive) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn/mmmmm + code |= insEncodeUimm<12, 10>(imm & 0b111); // iii + code |= insEncodeUimm<20, 16>(imm >> 3); // iiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BN_1A: // ............iiii ......pppppddddd -- SVE inc/dec vector by element count + case IF_SVE_BP_1A: // ............iiii ......pppppddddd -- SVE saturating inc/dec vector by element count + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeUimm4From1_19_to_16(imm); // iiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BS_1A: // ..............ii iiiiiiiiiiiddddd -- SVE bitwise logical with immediate (unpredicated) + case IF_SVE_BT_1A: // ..............ii iiiiiiiiiiiddddd -- SVE broadcast bitmask immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= (imm << 5); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BU_2A: // ........xx..gggg ...iiiiiiiiddddd -- SVE copy floating-point immediate (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BV_2A: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) + case IF_SVE_BV_2A_J: // ........xx..gggg ..hiiiiiiiiddddd -- SVE copy integer immediate (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= (id->idHasShift() ? 0x2000 : 0); // h + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BV_2B: // ........xx..gggg ...........ddddd -- SVE copy integer immediate (predicated) + // In emitIns, we set this format's instruction to MOV, as that is the preferred disassembly. + // However, passing (MOV, IF_SVE_BV_2B) to emitInsCodeSve will assert with "encoding_found", + // as FMOV is the only instruction associated with this encoding format. + // Thus, always pass FMOV here, and use MOV elsewhere for simplicity. + code = emitInsCodeSve(INS_sve_fmov, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_19_to_16(id->idReg2()); // gggg + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BW_2A: // ........ii.xxxxx ......nnnnnddddd -- SVE broadcast indexed element + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveBroadcastIndex(optGetSveElemsize(id->idInsOpt()), imm); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2A: // ................ ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2B: // .........i...ii. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2C: // ..............i. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CE_2D: // .............ii. ......nnnnn.DDDD -- SVE move predicate from vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2A: // ................ .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2B: // .........i...ii. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeSplitUimm<22, 22, 18, 17>(emitGetInsSC(id)); // i...ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2C: // ..............i. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeUimm<17, 17>(emitGetInsSC(id)); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CF_2D: // .............ii. .......NNNNddddd -- SVE move predicate into vector + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeUimm<18, 17>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CC_2A: // ........xx...... ......mmmmmddddd -- SVE insert SIMD&FP scalar register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CD_2A: // ........xx...... ......mmmmmddddd -- SVE insert general register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CI_3A: // ........xx..MMMM .......NNNN.DDDD -- SVE permute predicate elements + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeReg_P_19_to_16(id->idReg3()); // MMMM + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CJ_2A: // ........xx...... .......nnnn.dddd -- SVE reverse predicate elements + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CK_2A: // ................ .......NNNN.DDDD -- SVE unpack predicate elements + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GQ_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision odd elements + code = emitInsCodeSve(ins, fmt); + + if (ins == INS_sve_fcvtnt && id->idInsOpt() == INS_OPTS_D_TO_S) + { + code |= (1 << 22 | 1 << 17); + } + else if (ins == INS_sve_fcvtlt && id->idInsOpt() == INS_OPTS_S_TO_D) + { + code |= (1 << 22 | 1 << 17); + } + + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + // Scalable to general register. + case IF_SVE_CO_3A: // ........xx...... ...gggmmmmmddddd -- SVE conditionally extract element to general register + case IF_SVE_CS_3A: // ........xx...... ...gggnnnnnddddd -- SVE extract element to general register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_Rd(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + // Scalable from general register. + case IF_SVE_CQ_3A: // ........xx...... ...gggnnnnnddddd -- SVE copy general register to vector (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_Rn(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CT_3A: // ................ ...gggnnnnnddddd -- SVE reverse doublewords + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CV_3A: // ........xx...... ...VVVnnnnnddddd -- SVE vector splice (destructive) + case IF_SVE_CV_3B: // ........xx...... ...VVVmmmmmddddd -- SVE vector splice (destructive) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // VVV + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn/mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CW_4A: // ........xx.mmmmm ..VVVVnnnnnddddd -- SVE select vector elements (predicated) + { + regNumber reg4 = (ins == INS_sve_mov ? id->idReg1() : id->idReg4()); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_13_to_10(id->idReg2()); // VVVV + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(reg4); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_CX_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors + case IF_SVE_CX_4A_A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE integer compare vectors + case IF_SVE_GE_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE2 character match + case IF_SVE_HT_4A: // ........xx.mmmmm ...gggnnnnn.DDDD -- SVE floating-point compare vectors + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeReg_V_20_to_16(id->idReg4()); // nnnnn + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CY_3A: // ........xx.iiiii ...gggnnnnn.DDDD -- SVE integer compare with signed immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm<20, 16>(imm); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CY_3B: // ........xx.iiiii ii.gggnnnnn.DDDD -- SVE integer compare with unsigned immediate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm<20, 14>(imm); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EW_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 multiply-add (checked pointer) + case IF_SVE_BR_3B: // ...........mmmmm ......nnnnnddddd -- SVE permute vector segments + case IF_SVE_FN_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply long + case IF_SVE_FO_3A: // ...........mmmmm ......nnnnnddddd -- SVE integer matrix multiply accumulate + case IF_SVE_AT_3B: // ...........mmmmm ......nnnnnddddd -- SVE integer add/subtract vectors (unpredicated) + case IF_SVE_AU_3A: // ...........mmmmm ......nnnnnddddd -- SVE bitwise logical operations (unpredicated) + case IF_SVE_BD_3B: // ...........mmmmm ......nnnnnddddd -- SVE2 integer multiply vectors (unpredicated) + case IF_SVE_EF_3A: // ...........mmmmm ......nnnnnddddd -- SVE two-way dot product + case IF_SVE_EI_3A: // ...........mmmmm ......nnnnnddddd -- SVE mixed sign dot product + case IF_SVE_GJ_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 crypto constructive binary operations + case IF_SVE_GN_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long + case IF_SVE_GO_3A: // ...........mmmmm ......nnnnnddddd -- SVE2 FP8 multiply-add long long + case IF_SVE_GW_3B: // ...........mmmmm ......nnnnnddddd -- SVE FP clamp + case IF_SVE_HA_3A: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_E: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HA_3A_F: // ...........mmmmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product + case IF_SVE_HB_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating-point multiply-add long + case IF_SVE_HD_3A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HD_3A_A: // ...........mmmmm ......nnnnnddddd -- SVE floating point matrix multiply accumulate + case IF_SVE_HK_3B: // ...........mmmmm ......nnnnnddddd -- SVE floating-point arithmetic (unpredicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AV_3A: // ...........mmmmm ......kkkkkddddd -- SVE2 bitwise ternary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeReg_V_9_to_5(id->idReg3()); // kkkkk + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AW_2A: // ........xx.xxiii ......mmmmmddddd -- sve_int_rotate_imm + imm = insSveGetImmDiff(emitGetInsSC(id), id->idInsOpt()); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<20, 16>(imm & 0b11111); // xxiii + code |= insEncodeUimm<22, 22>(imm >> 5); // x + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AX_1A: // ........xx.iiiii ......iiiiiddddd -- SVE index generation (immediate start, immediate + // increment) + { + ssize_t imm1; + ssize_t imm2; + insSveDecodeTwoSimm5(emitGetInsSC(id), &imm1, &imm2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<9, 5>(imm1); // iiiii + code |= insEncodeSimm<20, 16>(imm2); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_AY_2A: // ........xx.mmmmm ......iiiiiddddd -- SVE index generation (immediate start, register + // increment) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<9, 5>(emitGetInsSC(id)); // iiiii + code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_AZ_2A: // ........xx.iiiii ......nnnnnddddd -- SVE index generation (register start, immediate + // increment) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_R_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeSimm<20, 16>(emitGetInsSC(id)); // iiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BB_2A: // ...........nnnnn .....iiiiiiddddd -- SVE stack frame adjustment + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii + code |= insEncodeReg_R_20_to_16(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BC_1A: // ................ .....iiiiiiddddd -- SVE stack frame size + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeSimm<10, 5>(emitGetInsSC(id)); // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EW_3B: // ...........mmmmm ......aaaaaddddd -- SVE2 multiply-add (checked pointer) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg3()); // aaaaa + code |= insEncodeReg_V_20_to_16(id->idReg2()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EG_3A: // ...........iimmm ......nnnnnddddd -- SVE two-way dot product (indexed) + case IF_SVE_EY_3A: // ...........iimmm ......nnnnnddddd -- SVE integer dot product (indexed) + case IF_SVE_EZ_3A: // ...........iimmm ......nnnnnddddd -- SVE mixed sign dot product (indexed) + case IF_SVE_FD_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) + case IF_SVE_FF_3B: // ...........iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) + case IF_SVE_FI_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) + case IF_SVE_GU_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3A: // ...........iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_GY_3B: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) + case IF_SVE_GY_3B_D: // ...........iimmm ......nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) + case IF_SVE_FK_3B: // ...........iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(emitGetInsSC(id)); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FD_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply (indexed) + case IF_SVE_FF_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) + case IF_SVE_FI_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) + case IF_SVE_GU_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3C: // .........i.iimmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_FK_3A: // .........i.iimmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(imm & 0b11); // ii + code |= insEncodeUimm<22, 22>(imm >> 2); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FE_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) + case IF_SVE_FG_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) + case IF_SVE_FH_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) + case IF_SVE_FJ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) + case IF_SVE_GY_3A: // ...........iimmm ....i.nnnnnddddd -- SVE BFloat16 floating-point dot product (indexed) + case IF_SVE_GZ_3A: // ...........iimmm ....i.nnnnnddddd -- SVE floating-point multiply-add long (indexed) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 11>(imm & 1); // i + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(imm >> 1); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FE_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply long (indexed) + case IF_SVE_FG_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 integer multiply-add long (indexed) + case IF_SVE_FH_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply (indexed) + case IF_SVE_FJ_3B: // ...........immmm ....i.nnnnnddddd -- SVE2 saturating multiply-add (indexed) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 11>(imm & 1); // i + code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm + code |= insEncodeUimm<20, 19>(imm & 0b10); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EY_3B: // ...........immmm ......nnnnnddddd -- SVE integer dot product (indexed) + case IF_SVE_FD_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply (indexed) + case IF_SVE_FF_3C: // ...........immmm ......nnnnnddddd -- SVE2 integer multiply-add (indexed) + case IF_SVE_FI_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply high (indexed) + case IF_SVE_GU_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply-add (indexed) + case IF_SVE_GX_3B: // ...........immmm ......nnnnnddddd -- SVE floating-point multiply (indexed) + case IF_SVE_FK_3C: // ...........immmm ......nnnnnddddd -- SVE2 saturating multiply-add high (indexed) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm + + // index is encoded at bit location 20; + // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 + code |= insEncodeUimm<20, 19>(emitGetInsSC(id) << 1); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CZ_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + case IF_SVE_DA_4A: // ............MMMM ..gggg.NNNN.DDDD -- SVE propagate break from previous partition + { + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + + regNumber regm; + switch (ins) + { + case INS_sve_mov: + case INS_sve_movs: + regm = id->idReg3(); + break; + + case INS_sve_not: + case INS_sve_nots: + regm = id->idReg2(); + break; + + default: + regm = id->idReg4(); + } + + code |= insEncodeReg_P_19_to_16(regm); // MMMM + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_CZ_4A_A: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + case IF_SVE_CZ_4A_L: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeReg_P_19_to_16(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CZ_4A_K: // ............MMMM ..gggg.NNNN.DDDD -- SVE predicate logical operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodeReg_P_19_to_16(id->idReg1()); // DDDD + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DB_3A: // ................ ..gggg.NNNNMDDDD -- SVE partition break condition + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodePredQualifier_4(id->idPredicateReg2Merge()); // M + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DB_3B: // ................ ..gggg.NNNN.DDDD -- SVE partition break condition + case IF_SVE_DC_3A: // ................ ..gggg.NNNN.MMMM -- SVE propagate break to next partition + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DD_2A: // ................ .......gggg.DDDD -- SVE predicate first active + case IF_SVE_DG_2A: // ................ .......gggg.DDDD -- SVE predicate read from FFR (predicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // gggg + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DE_1A: // ........xx...... ......ppppp.DDDD -- SVE predicate initialize + code = emitInsCodeSve(ins, fmt); + code |= insEncodeSvePattern(id->idSvePattern()); // ppppp + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DF_2A: // ........xx...... .......VVVV.DDDD -- SVE predicate next active + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_8_to_5(id->idReg2()); // VVVV + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DH_1A: // ................ ............DDDD -- SVE predicate read from FFR (unpredicated) + case IF_SVE_DJ_1A: // ................ ............DDDD -- SVE predicate zero + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DI_2A: // ................ ..gggg.NNNN..... -- SVE predicate test + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_13_to_10(id->idReg1()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DK_3A: // ........xx...... ..gggg.NNNNddddd -- SVE predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_13_to_10(id->idReg2()); // gggg + code |= insEncodeReg_P_8_to_5(id->idReg3()); // NNNN + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GA_2A: // ............iiii ......nnnn.ddddd -- SME2 multi-vec shift narrow + imm = emitGetInsSC(id); + assert(id->idInsOpt() == INS_OPTS_SCALABLE_H); + assert(emitInsIsVectorRightShift(id->idIns())); + assert(isValidVectorShiftAmount(imm, EA_4BYTE, /* rightShift */ true)); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeVectorShift(EA_4BYTE, true /* right-shift */, imm); // iiii + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_6_Times_Two(id->idReg2()); // nnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DL_2A: // ........xx...... .....l.NNNNddddd -- SVE predicate count (predicate-as-counter) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeVectorLengthSpecifier(id); // l + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // NNNN + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DM_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec register by predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DN_2A: // ........xx...... .......MMMMddddd -- SVE inc/dec vector by predicate count + case IF_SVE_DP_2A: // ........xx...... .......MMMMddddd -- SVE saturating inc/dec vector by predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DO_2A: // ........xx...... .....X.MMMMddddd -- SVE saturating inc/dec register by predicate count + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_8_to_5(id->idReg2()); // MMMM + code |= insEncodeVLSElemsize(id->idOpSize()); // X + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DQ_0A: // ................ ................ -- SVE FFR initialise + code = emitInsCodeSve(ins, fmt); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DR_1A: // ................ .......NNNN..... -- SVE FFR write from predicate + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_8_to_5(id->idReg1()); // NNNN + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DS_2A: // .........x.mmmmm ......nnnnn..... -- SVE conditionally terminate scalars + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_R_9_to_5(id->idReg1()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg2()); // mmmmm + code |= insEncodeSveElemsize_R_22(id->idOpSize()); // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FZ_2A: // ................ ......nnnn.ddddd -- SME2 multi-vec extract narrow + case IF_SVE_HG_2A: // ................ ......nnnn.ddddd -- SVE2 FP8 downconverts + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_6(id->idReg2()); // nnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GD_2A: // .........x.xx... ......nnnnnddddd -- SVE2 saturating extract narrow + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + // Bit 23 should not be set by below call + assert(insOptsScalableWide(id->idInsOpt())); + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FR_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift left long + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<20, 16>(emitGetInsSC(id)); // iii + // Bit 23 should not be set by below call + assert(insOptsScalableWide(id->idInsOpt())); + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GB_2A: // .........x.xxiii ......nnnnnddddd -- SVE2 bitwise shift right narrow + // Bit 23 should not be set by call to insEncodeSveElemsize_tszh_23_tszl_20_to_19, + // nor should we pass INS_OPTS_SCALABLE_D to insGetImmDiff. + assert(insOptsScalableWide(id->idInsOpt())); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<20, 16>(insSveGetImmDiff(emitGetInsSC(id), id->idInsOpt())); // iii + code |= insEncodeSveElemsize_tszh_23_tszl_20_to_19(optGetSveElemsize(id->idInsOpt())); // xx + // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FV_2A: // ........xx...... .....rmmmmmddddd -- SVE2 complex integer add + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<10, 10>(emitGetInsSC(id)); // r + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FY_3A: // .........x.mmmmm ......nnnnnddddd -- SVE2 integer add/subtract long with carry + { + // Size encoding: 1 if INS_OPTS_SCALABLE_D, 0 if INS_OPTS_SCALABLE_S + const ssize_t sizeEncoding = (id->idInsOpt() == INS_OPTS_SCALABLE_D) ? 1 : 0; + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<22, 22>(sizeEncoding); // x + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_GK_2A: // ................ ......mmmmmddddd -- SVE2 crypto destructive binary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GL_1A: // ................ ...........ddddd -- SVE2 crypto unary operations + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DT_3A: // ........xx.mmmmm ...X..nnnnn.DDDD -- SVE integer compare scalar count and limit + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= (id->idOpSize() == EA_8BYTE) ? (1 << 12) : 0; // X + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DW_2A: // ........xx...... ......iiNNN.DDDD -- SVE extract mask predicate from predicate-as-counter + case IF_SVE_DW_2B: // ........xx...... .......iNNN.DDDD -- SVE extract mask predicate from predicate-as-counter + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_7_to_5(id->idReg2()); // NNN + code |= insEncodeUimm<9, 8>(emitGetInsSC(id)); // ii (or i) + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DX_3A: // ........xx.mmmmm ......nnnnn.DDD. -- SVE integer compare scalar count and limit (predicate + // pair) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_1(id->idReg1()); // DDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DY_3A: // ........xx.mmmmm ..l...nnnnn..DDD -- SVE integer compare scalar count and limit + // (predicate-as-counter) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeVectorLengthSpecifier(id); // l + code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DZ_1A: // ........xx...... .............DDD -- sve_int_pn_ptrue + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_2_to_0(id->idReg1()); // DDD + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EA_1A: // ........xx...... ...iiiiiiiiddddd -- SVE broadcast floating-point immediate (unpredicated) + case IF_SVE_ED_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer min/max immediate (unpredicated) + case IF_SVE_EE_1A: // ........xx...... ...iiiiiiiiddddd -- SVE integer multiply immediate (unpredicated) + { + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_FA_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) + case IF_SVE_FB_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) + case IF_SVE_FC_3A: // ...........iimmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) + { + const ssize_t imm = emitGetInsSC(id); + const ssize_t rot = (imm & 0b11); + const ssize_t index = (imm >> 2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 10>(rot); // rr + code |= insEncodeReg_V_18_to_16(id->idReg3()); // mmm + code |= insEncodeUimm<20, 19>(index); // ii + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_EJ_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer dot product + case IF_SVE_EK_3A: // ........xx.mmmmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeUimm<11, 10>(emitGetInsSC(id)); // rr + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_FA_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer dot product (indexed) + case IF_SVE_FB_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex integer multiply-add (indexed) + case IF_SVE_FC_3B: // ...........immmm ....rrnnnnnddddd -- SVE2 complex saturating multiply-add (indexed) + case IF_SVE_GV_3A: // ...........immmm ....rrnnnnnddddd -- SVE floating-point complex multiply-add (indexed) + { + const ssize_t imm = emitGetInsSC(id); + const ssize_t rot = (imm & 0b11); + const ssize_t index = (imm >> 2); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_19_to_16(id->idReg3()); // mmmm + code |= insEncodeUimm<11, 10>(rot); // rr + + // index is encoded at bit location 20; + // left-shift by one bit so we can reuse insEncodeUimm<20, 19> without modifying bit location 19 + code |= insEncodeUimm<20, 19>(index << 1); // i + dst += emitOutput_Instr(dst, code); + break; + } + + case IF_SVE_EB_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE broadcast integer immediate (unpredicated) + case IF_SVE_EC_1A: // ........xx...... ..hiiiiiiiiddddd -- SVE integer add/subtract immediate (unpredicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + code |= insEncodeImm8_12_to_5(imm); // iiiiiiii + code |= (id->idHasShift() ? 0x2000 : 0); // h + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_EB_1B: // ........xx...... ...........ddddd -- SVE broadcast integer immediate (unpredicated) + // ins is MOV for this encoding, as it is the preferred disassembly, so pass FMOV to emitInsCodeSve + code = emitInsCodeSve(INS_sve_fmov, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DU_3A: // ........xx.mmmmm ......nnnnn.DDDD -- SVE pointer conflict compare + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_DV_4A: // ........ix.xxxvv ..NNNN.MMMM.DDDD -- SVE broadcast predicate element + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_13_to_10(id->idReg2()); // NNNN + code |= insEncodeReg_P_8_to_5(id->idReg3()); // MMMM + code |= insEncodeReg_R_17_to_16(id->idReg4()); // vv + code |= insEncodeSveElemsize_tszh_tszl_and_imm(id->idInsOpt(), emitGetInsSC(id)); // ix xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HO_3A: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HO_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + switch (id->idInsOpt()) + { + case INS_OPTS_H_TO_S: + code |= (1 << 16); + break; + case INS_OPTS_H_TO_D: + code |= (1 << 22) | (1 << 16); + break; + case INS_OPTS_S_TO_H: + break; + case INS_OPTS_S_TO_D: + code |= (1 << 22) | (3 << 16); + break; + case INS_OPTS_D_TO_H: + code |= (1 << 22); + break; + case INS_OPTS_D_TO_S: + code |= (1 << 22) | (1 << 17); + break; + default: + unreached(); + } + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HO_3C: // ................ ...gggnnnnnddddd -- SVE floating-point convert precision + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HP_3B: // ................ ...gggnnnnnddddd -- SVE floating-point convert to integer + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_H: + code |= (1 << 22) | (1 << 17); + break; + case INS_OPTS_H_TO_S: + code |= (1 << 22) | (1 << 18); + break; + case INS_OPTS_H_TO_D: + code |= (1 << 22) | (3 << 17); + break; + case INS_OPTS_SCALABLE_S: + code |= (1 << 23) | (1 << 18); + break; + case INS_OPTS_S_TO_D: + code |= (3 << 22) | (1 << 18); + break; + case INS_OPTS_D_TO_S: + code |= (3 << 22); + break; + case INS_OPTS_SCALABLE_D: + code |= (3 << 22) | (3 << 17); + break; + default: + unreached(); + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HS_3A: // ................ ...gggnnnnnddddd -- SVE integer convert to floating-point + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_H: + code |= (1 << 22) | (1 << 17); + break; + case INS_OPTS_S_TO_H: + code |= (1 << 22) | (1 << 18); + break; + case INS_OPTS_SCALABLE_S: + code |= (1 << 23) | (1 << 18); + break; + case INS_OPTS_S_TO_D: + code |= (1 << 23) | (1 << 22); + break; + case INS_OPTS_D_TO_H: + code |= (1 << 22) | (3 << 17); + break; + case INS_OPTS_D_TO_S: + code |= (3 << 22) | (1 << 18); + break; + case INS_OPTS_SCALABLE_D: + code |= (3 << 22) | (3 << 17); + break; + default: + unreached(); + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IH_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IH_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus + // immediate) + case IF_SVE_IJ_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_E: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_F: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IJ_3A_G: // ............iiii ...gggnnnnnttttt -- SVE contiguous load (scalar plus immediate) + case IF_SVE_IL_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus immediate) + case IF_SVE_IL_3A_A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_B: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IL_3A_C: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-fault load (scalar plus + // immediate) + case IF_SVE_IM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus + // immediate) + case IF_SVE_IO_3A: // ............iiii ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus + // immediate) + case IF_SVE_IQ_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_IS_3A: // ............iiii ...gggnnnnnttttt -- SVE load multiple structures (scalar plus immediate) + case IF_SVE_JE_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // immediate) + case IF_SVE_JM_3A: // ............iiii ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // immediate) + case IF_SVE_JN_3C: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JN_3C_D: // ............iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + case IF_SVE_JO_3A: // ............iiii ...gggnnnnnttttt -- SVE store multiple structures (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + + switch (ins) + { + case INS_sve_ld2b: + case INS_sve_ld2h: + case INS_sve_ld2w: + case INS_sve_ld2d: + case INS_sve_ld2q: + case INS_sve_st2b: + case INS_sve_st2h: + case INS_sve_st2w: + case INS_sve_st2d: + case INS_sve_st2q: + code |= insEncodeSimm4_MultipleOf2_19_to_16(imm); // iiii + break; + + case INS_sve_ld3b: + case INS_sve_ld3h: + case INS_sve_ld3w: + case INS_sve_ld3d: + case INS_sve_ld3q: + case INS_sve_st3b: + case INS_sve_st3h: + case INS_sve_st3w: + case INS_sve_st3d: + case INS_sve_st3q: + code |= insEncodeSimm4_MultipleOf3_19_to_16(imm); // iiii + break; + + case INS_sve_ld4b: + case INS_sve_ld4h: + case INS_sve_ld4w: + case INS_sve_ld4d: + case INS_sve_ld4q: + case INS_sve_st4b: + case INS_sve_st4h: + case INS_sve_st4w: + case INS_sve_st4d: + case INS_sve_st4q: + code |= insEncodeSimm4_MultipleOf4_19_to_16(imm); // iiii + break; + + case INS_sve_ld1rqb: + case INS_sve_ld1rqd: + case INS_sve_ld1rqh: + case INS_sve_ld1rqw: + code |= insEncodeSimm4_MultipleOf16_19_to_16(imm); // iiii + break; + + case INS_sve_ld1rob: + case INS_sve_ld1rod: + case INS_sve_ld1roh: + case INS_sve_ld1row: + code |= insEncodeSimm4_MultipleOf32_19_to_16(imm); // iiii + break; + + default: + code |= insEncodeSimm<19, 16>(imm); // iiii + break; + } + + if (canEncodeSveElemsize_dtype(ins)) + { + if (ins == INS_sve_ld1w) + { + code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + } + else + { + code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); + } + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JD_4A: // .........xxmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JD_4B: // ..........xmmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JJ_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // // offsets) + case IF_SVE_JJ_4A_C: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4A_D: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JK_4A: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled + // offsets) + case IF_SVE_JK_4A_B: // ...........mmmmm .h.gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit + // unscaled offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + code |= (1 << 14); // h + break; + + default: + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JN_3A: // .........xx.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm<19, 16>(imm); // iiii + code |= insEncodeSveElemsize_22_to_21(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JN_3B: // ..........x.iiii ...gggnnnnnttttt -- SVE contiguous store (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSimm<19, 16>(imm); // iiii + code |= insEncodeSveElemsize_sz_21(optGetSveElemsize(id->idInsOpt())); // x + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HW_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4A_B: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_IU_4A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4A_A: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4A_C: // .........h.mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + code |= (1 << 22); // h + break; + + default: + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HW_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + case IF_SVE_HW_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 32-bit gather load (scalar plus 32-bit unscaled + // offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus + // scalar) + case IF_SVE_IF_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit gather non-temporal load (vector plus + // scalar) + case IF_SVE_IW_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit gather load (vector plus scalar) + case IF_SVE_IX_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit gather non-temporal load (vector plus + // scalar) + case IF_SVE_IY_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 128-bit scatter store (vector plus scalar) + case IF_SVE_IZ_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus + // scalar) + case IF_SVE_IZ_4A_A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 32-bit scatter non-temporal store (vector plus + // scalar) + case IF_SVE_JA_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE2 64-bit scatter non-temporal store (vector plus + // scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IG_4A_D: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_IG_4A_E: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_IG_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_IG_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus + // scalar) + case IF_SVE_II_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) + case IF_SVE_IK_4A_F: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IK_4A_G: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IK_4A_H: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IK_4A_I: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + + if (canEncodeSveElemsize_dtype(ins)) + { + if (ins == INS_sve_ld1w) + { + code = insEncodeSveElemsize_dtype_ld1w(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + } + else + { + code = insEncodeSveElemsize_dtype(ins, optGetSveElemsize(id->idInsOpt()), code); + } + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IG_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous first-fault load (scalar plus scalar) + case IF_SVE_II_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) + case IF_SVE_II_4A_B: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (quadwords, scalar plus scalar) + case IF_SVE_IK_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous load (scalar plus scalar) + case IF_SVE_IN_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal load (scalar plus scalar) + case IF_SVE_IP_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load and broadcast quadword (scalar plus scalar) + case IF_SVE_IR_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (quadwords, scalar plus + // scalar) + case IF_SVE_IT_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE load multiple structures (scalar plus scalar) + case IF_SVE_JB_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous non-temporal store (scalar plus + // scalar) + case IF_SVE_JC_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (scalar plus scalar) + case IF_SVE_JD_4C: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + case IF_SVE_JD_4C_A: // ...........mmmmm ...gggnnnnnttttt -- SVE contiguous store (scalar plus scalar) + case IF_SVE_JF_4A: // ...........mmmmm ...gggnnnnnttttt -- SVE store multiple structures (quadwords, scalar plus + // scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IU_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4B_B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_IU_4B_D: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit gather load (scalar plus 32-bit unpacked + // scaled offsets) + case IF_SVE_JJ_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4B_C: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JJ_4B_E: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit scaled + // offsets) + case IF_SVE_JK_4B: // ...........mmmmm ...gggnnnnnttttt -- SVE 64-bit scatter store (scalar plus 64-bit unscaled + // offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GP_3A: // ........xx.....r ...gggmmmmmddddd -- SVE floating-point complex add (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeSveImm90_or_270_rot(imm); // r + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GT_4A: // ........xx.mmmmm .rrgggnnnnnddddd -- SVE floating-point complex multiply-add (predicated) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + code |= insEncodeSveImm0_to_270_rot(imm); // rr + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HI_3A: // ........xx...... ...gggnnnnn.DDDD -- SVE floating-point compare with zero + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // DDDD + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HM_2A: // ........xx...... ...ggg....iddddd -- SVE floating-point arithmetic with immediate + // (predicated) + { + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeSveSmallFloatImm(imm); // i + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + } + break; + + case IF_SVE_HN_2A: // ........xx...iii ......mmmmmddddd -- SVE floating-point trig multiply-add coefficient + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<18, 16>(imm); // iii + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HP_3A: // .............xx. ...gggnnnnnddddd -- SVE floating-point convert to integer + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize_18_to_17(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HU_4B: // ...........mmmmm ...gggnnnnnddddd -- SVE floating-point multiply-accumulate writing addend + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg4()); // mmmmm + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HV_4A: // ........xx.aaaaa ...gggmmmmmddddd -- SVE floating-point multiply-accumulate writing + // multiplicand + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // mmmmm + code |= insEncodeReg_V_20_to_16(id->idReg4()); // aaaaa + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_ID_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE load predicate register + case IF_SVE_JG_2A: // ..........iiiiii ...iiinnnnn.TTTT -- SVE store predicate register + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_3_to_0(id->idReg1()); // TTTT + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii + // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IE_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE load vector register + case IF_SVE_JH_2A: // ..........iiiiii ...iiinnnnnttttt -- SVE store vector register + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSimm9h9l_21_to_16_and_12_to_10(imm); // iii + // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GG_3A: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + case IF_SVE_GH_3B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + case IF_SVE_GH_3B_B: // ........ii.mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm<23, 22>(imm); // ii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GG_3B: // ........ii.mmmmm ...i..nnnnnddddd -- SVE2 lookup table with 2-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm3h3l_23_to_22_and_12(imm); // ii + // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_GH_3A: // ........i..mmmmm ......nnnnnddddd -- SVE2 lookup table with 4-bit indices and 16-bit + // element size + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= insEncodeUimm1_23(imm); // i + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HY_3A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled + // offsets) + case IF_SVE_HY_3A_A: // .........h.mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit + // scaled offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= id->idSvePrfop(); // oooo + + switch (id->idInsOpt()) + { + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + code |= (1 << 22); // h + break; + + default: + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HY_3B: // ...........mmmmm ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (scalar plus 32-bit scaled + // offsets) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_V_20_to_16(id->idReg3()); // mmmmm + code |= id->idSvePrfop(); // oooo + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IB_3A: // ...........mmmmm ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus scalar) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeReg_R_20_to_16(id->idReg3()); // mmmmm + code |= id->idSvePrfop(); // oooo + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HZ_2A_B: // ...........iiiii ...gggnnnnn.oooo -- SVE 32-bit gather prefetch (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= id->idSvePrfop(); // oooo + + if (id->idInsOpt() == INS_OPTS_SCALABLE_D) + { + code |= (1 << 30); // set bit '30' to make it a double-word + } + + switch (ins) + { + case INS_sve_prfh: + code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii + break; + + case INS_sve_prfw: + code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii + break; + + case INS_sve_prfd: + code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii + break; + + default: + assert(ins == INS_sve_prfb); + } + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HX_3A_B: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm<20, 16>(imm); // iiiii + code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_HX_3A_E: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit gather load (vector plus immediate) + case IF_SVE_IV_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit gather load (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + + switch (ins) + { + case INS_sve_ld1d: + case INS_sve_ldff1d: + code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii + break; + + case INS_sve_ld1w: + case INS_sve_ld1sw: + case INS_sve_ldff1w: + case INS_sve_ldff1sw: + code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii + break; + + default: + code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JL_3A: // ...........iiiii ...gggnnnnnttttt -- SVE 64-bit scatter store (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeUimm5_MultipleOf8_20_to_16(imm); // iiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_JI_3A_A: // ...........iiiii ...gggnnnnnttttt -- SVE 32-bit scatter store (vector plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_V_9_to_5(id->idReg3()); // nnnnn + code |= insEncodeSveElemsize_30_or_21(fmt, optGetSveElemsize(id->idInsOpt())); + + switch (ins) + { + case INS_sve_st1h: + code |= insEncodeUimm5_MultipleOf2_20_to_16(imm); // iiiii + break; + + case INS_sve_st1w: + code |= insEncodeUimm5_MultipleOf4_20_to_16(imm); // iiiii + break; + + default: + assert(ins == INS_sve_st1b); + code |= insEncodeUimm<20, 16>(imm); // iiiii + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IA_2A: // ..........iiiiii ...gggnnnnn.oooo -- SVE contiguous prefetch (scalar plus immediate) + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_P_12_to_10(id->idReg1()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg2()); // nnnnn + code |= id->idSvePrfop(); // oooo + code |= insEncodeSimm<21, 16>(imm); // iiiiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IC_3A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + + switch (ins) + { + case INS_sve_ld1rd: + code |= insEncodeUimm6_MultipleOf8_21_to_16(imm); // iiiiii + break; + + default: + assert(ins == INS_sve_ld1rsw); + code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_IC_3A_A: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + case IF_SVE_IC_3A_B: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + case IF_SVE_IC_3A_C: // ..........iiiiii ...gggnnnnnttttt -- SVE load and broadcast element + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ttttt + code |= insEncodeReg_P_12_to_10(id->idReg2()); // ggg + code |= insEncodeReg_R_9_to_5(id->idReg3()); // nnnnn + code = insEncodeSveElemsize_dtypeh_dtypel(ins, fmt, optGetSveElemsize(id->idInsOpt()), code); + + switch (ins) + { + case INS_sve_ld1rw: + code |= insEncodeUimm6_MultipleOf4_21_to_16(imm); // iiiiii + break; + + case INS_sve_ld1rh: + case INS_sve_ld1rsh: + code |= insEncodeUimm6_MultipleOf2_21_to_16(imm); // iiiiii + break; + + default: + code |= insEncodeUimm<21, 16>(imm); // iiiiii + break; + } + + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BI_2A: // ................ ......nnnnnddddd -- SVE constructive prefix (unpredicated) + case IF_SVE_HH_2A: // ................ ......nnnnnddddd -- SVE2 FP8 upconverts + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_CB_2A: // ........xx...... ......nnnnnddddd -- SVE broadcast general register + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_Rn(id->idReg2()); // nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BJ_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point exponential accelerator + case IF_SVE_CG_2A: // ........xx...... ......nnnnnddddd -- SVE reverse vector elements + case IF_SVE_CH_2A: // ........xx...... ......nnnnnddddd -- SVE unpack vector elements + case IF_SVE_HF_2A: // ........xx...... ......nnnnnddddd -- SVE floating-point reciprocal estimate (unpredicated) + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsize(optGetSveElemsize(id->idInsOpt())); // xx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BF_2A: // ........xx.xxiii ......nnnnnddddd -- SVE bitwise shift by immediate (unpredicated) + case IF_SVE_FT_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift and insert + case IF_SVE_FU_2A: // ........xx.xxiii ......nnnnnddddd -- SVE2 bitwise shift right and accumulate + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsizeWithShift_tszh_tszl_imm3(id->idInsOpt(), imm, + emitInsIsVectorRightShift(ins)); // xx xxiii + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BX_2A: // ...........ixxxx ......nnnnnddddd -- sve_int_perm_dupq_i + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // nnnnn + code |= insEncodeSveElemsizeWithImmediate_i1_tsz(id->idInsOpt(), imm); // ixxxx + dst += emitOutput_Instr(dst, code); + break; + + case IF_SVE_BY_2A: // ............iiii ......mmmmmddddd -- sve_int_perm_extq + imm = emitGetInsSC(id); + code = emitInsCodeSve(ins, fmt); + code |= insEncodeReg_V_4_to_0(id->idReg1()); // ddddd + code |= insEncodeReg_V_9_to_5(id->idReg2()); // mmmmm + code |= insEncodeUimm<19, 16>(imm); // iiii + dst += emitOutput_Instr(dst, code); + break; + + default: + assert(!"Unexpected format"); + break; + } + + return dst; +} + +/***************************************************************************** + * + * Prints the encoding for the Extend Type encoding + */ + +void emitter::emitDispSveExtendOpts(insOpts opt) +{ + switch (opt) + { + case INS_OPTS_LSL: + printf("lsl"); + break; + + case INS_OPTS_UXTW: + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_D_UXTW: + printf("uxtw"); + break; + + case INS_OPTS_SXTW: + case INS_OPTS_SCALABLE_S_SXTW: + case INS_OPTS_SCALABLE_D_SXTW: + printf("sxtw"); + break; + + default: + assert(!"Bad value"); + break; + } +} + +/***************************************************************************** + * + * Prints the encoding for the Extend Type encoding along with the N value + */ + +void emitter::emitDispSveExtendOptsModN(insOpts opt, ssize_t imm) +{ + assert(imm >= 0 && imm <= 3); + + if (imm == 0 && opt != INS_OPTS_LSL) + { + emitDispSveExtendOpts(opt); + } + else if (imm > 0) + { + emitDispSveExtendOpts(opt); + printf(" #%d", (int)imm); + } +} + +/***************************************************************************** + * + * Prints the encoding for the or LSL encoding along with the N value + * This is for formats that have [, .T, ], [, .T, #N], [, , LSL #N], + * [{, , LSL #N}] + */ +void emitter::emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts opt, insFormat fmt) +{ + printf("["); + + if (isVectorRegister(reg1)) + { + // If the overall instruction is working on 128-bit + // registers, the size of this register for + // the mod addr is always 64-bit. + // Example: LD1Q {.Q }, /Z, [.D{, }] + if (opt == INS_OPTS_SCALABLE_Q) + { + emitDispSveReg(reg1, INS_OPTS_SCALABLE_D, reg2 != REG_ZR); + } + else + { + emitDispSveReg(reg1, opt, reg2 != REG_ZR); + } + } + else + { + emitDispReg(reg1, EA_8BYTE, reg2 != REG_ZR); + } + + if (isVectorRegister(reg2)) + { + emitDispSveReg(reg2, opt, false); + } + else if (reg2 != REG_ZR) + { + emitDispReg(reg2, EA_8BYTE, false); + } + + if (insOptsScalable32bitExtends(opt)) + { + emitDispComma(); + emitDispSveExtendOptsModN(opt, insSveGetLslOrModN(ins, fmt)); + } + // Omit 'lsl #N' only if the second register is ZR. + else if ((reg2 != REG_ZR) && insSveIsLslN(ins, fmt)) + { + emitDispComma(); + switch (insSveGetLslOrModN(ins, fmt)) + { + case 4: + printf("lsl #4"); + break; + + case 3: + printf("lsl #3"); + break; + + case 2: + printf("lsl #2"); + break; + + case 1: + printf("lsl #1"); + break; + + default: + assert(!"Invalid instruction"); + break; + } + } + printf("]"); +} + +/***************************************************************************** + * + * Prints the encoding for format [.S{, #}] + */ +void emitter::emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt) +{ + printf("["); + emitDispSveReg(reg1, opt, imm != 0); + if (imm != 0) + { + // This does not have to be printed as hex. + // We only do it because the capstone disassembly displays this immediate as hex. + // We could not modify capstone without affecting other cases. + emitDispImm(imm, false, /* alwaysHex */ true); + } + printf("]"); +} + +/***************************************************************************** + * + * Prints the encoding for format [{, #, MUL VL}] + */ +void emitter::emitDispSveImmMulVl(regNumber reg1, ssize_t imm) +{ + printf("["); + emitDispReg(reg1, EA_8BYTE, imm != 0); + if (imm != 0) + { + emitDispImm(imm, true); + printf("mul vl"); + } + printf("]"); +} + +/***************************************************************************** + * + * Prints the encoding for format [.D{, #}] + */ +void emitter::emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm) +{ + printf("["); + if (isVectorRegister(reg1)) + { + emitDispSveReg(reg1, opt, imm != 0); + } + else + { + emitDispReg(reg1, EA_8BYTE, imm != 0); + } + if (imm != 0) + { + // This does not have to be printed as hex. + // We only do it because the capstone disassembly displays this immediate as hex. + // We could not modify capstone without affecting other cases. + emitDispImm(imm, false, /* alwaysHex */ (imm > 31)); + } + printf("]"); +} + +//------------------------------------------------------------------------ +// emitDispSveReg: Display a scalable vector register name +// +void emitter::emitDispSveReg(regNumber reg, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + + if (addComma) + emitDispComma(); +} + +//------------------------------------------------------------------------ +// emitDispSveReg: Display a scalable vector register name with an arrangement suffix +// +void emitter::emitDispSveReg(regNumber reg, insOpts opt, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + + if (opt != INS_OPTS_NONE) + { + assert(insOptsScalable(opt) || insOptsScalable32bitExtends(opt)); + emitDispArrangement(opt); + } + + if (addComma) + emitDispComma(); +} + +//------------------------------------------------------------------------ +// emitDispSveRegIndex: Display a scalable vector register with indexed element +// +void emitter::emitDispSveRegIndex(regNumber reg, ssize_t index, bool addComma) +{ + assert(isVectorRegister(reg)); + printf(emitSveRegName(reg)); + emitDispElementIndex(index, addComma); +} + +//------------------------------------------------------------------------ +// emitDispSveConsecutiveRegList: Display a SVE consecutive vector register list +// +void emitter::emitDispSveConsecutiveRegList(regNumber firstReg, unsigned listSize, insOpts opt, bool addComma) +{ + assert(isVectorRegister(firstReg)); + + regNumber currReg = firstReg; + + assert(listSize > 0); + + printf("{ "); + // We do not want the short-hand for list size of 1 or 2. + if ((listSize <= 2) || (((unsigned)currReg + listSize - 1) > (unsigned)REG_V31)) + { + for (unsigned i = 0; i < listSize; i++) + { + const bool notLastRegister = (i != listSize - 1); + emitDispSveReg(currReg, opt, notLastRegister); + currReg = (currReg == REG_V31) ? REG_V0 : REG_NEXT(currReg); + } + } + else + { + // short-hand. example: { z0.s - z2.s } which is the same as { z0.s, z1.s, z2.s } + emitDispSveReg(currReg, opt, false); + printf(" - "); + emitDispSveReg((regNumber)(currReg + listSize - 1), opt, false); + } + printf(" }"); + + if (addComma) + { + emitDispComma(); + } +} + +//------------------------------------------------------------------------ +// emitDispPredicateReg: Display a predicate register name with with an arrangement suffix +// +void emitter::emitDispPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) +{ + assert(isPredicateRegister(reg)); + printf(emitPredicateRegName(reg, ptype)); + + if (ptype == PREDICATE_MERGE) + { + printf("/m"); + } + else if (ptype == PREDICATE_ZERO) + { + printf("/z"); + } + else if (ptype == PREDICATE_SIZED || ptype == PREDICATE_N_SIZED) + { + emitDispElemsize(optGetSveElemsize(opt)); + } + + if (addComma) + emitDispComma(); +} + +//------------------------------------------------------------------------ +// emitDispPredicateRegPair: Display a pair of predicate registers +// +void emitter::emitDispPredicateRegPair(regNumber reg, insOpts opt) +{ + printf("{ "); + emitDispPredicateReg(reg, PREDICATE_SIZED, opt, true); + emitDispPredicateReg((regNumber)((unsigned)reg + 1), PREDICATE_SIZED, opt, false); + printf(" }, "); +} + +//------------------------------------------------------------------------ +// emitDispLowPredicateReg: Display a low predicate register name with with an arrangement suffix +// +void emitter::emitDispLowPredicateReg(regNumber reg, PredicateType ptype, insOpts opt, bool addComma) +{ + assert(isLowPredicateRegister(reg)); + reg = (regNumber)((((unsigned)reg - REG_PREDICATE_FIRST) & 0x7) + REG_PREDICATE_FIRST); + emitDispPredicateReg(reg, ptype, opt, addComma); +} + +//------------------------------------------------------------------------ +// emitDispLowPredicateRegPair: Display a pair of low predicate registers +// +void emitter::emitDispLowPredicateRegPair(regNumber reg, insOpts opt) +{ + assert(isLowPredicateRegister(reg)); + + printf("{ "); + const unsigned baseRegNum = ((unsigned)reg - REG_PREDICATE_FIRST) & 0x7; + const unsigned regNum = (baseRegNum * 2) + REG_PREDICATE_FIRST; + emitDispPredicateReg((regNumber)regNum, PREDICATE_SIZED, opt, true); + emitDispPredicateReg((regNumber)(regNum + 1), PREDICATE_SIZED, opt, false); + printf(" }, "); +} + +//------------------------------------------------------------------------ +// emitDispVectorLengthSpecifier: Display the vector length specifier +// +void emitter::emitDispVectorLengthSpecifier(instrDesc* id) +{ + assert(id != nullptr); + assert(insOptsScalableStandard(id->idInsOpt())); + + if (id->idVectorLength4x()) + { + printf("vlx4"); + } + else + { + printf("vlx2"); + } +} + +/***************************************************************************** + * + * Display an insSvePattern + */ +void emitter::emitDispSvePattern(insSvePattern pattern, bool addComma) +{ + printf("%s", svePatternNames[pattern]); + + if (addComma) + { + emitDispComma(); + } +} + +/***************************************************************************** + * + * Display an insSvePrfop + */ +void emitter::emitDispSvePrfop(insSvePrfop prfop, bool addComma) +{ + switch (prfop) + { + case SVE_PRFOP_PLDL1KEEP: + printf("pldl1keep"); + break; + + case SVE_PRFOP_PLDL1STRM: + printf("pldl1strm"); + break; + + case SVE_PRFOP_PLDL2KEEP: + printf("pldl2keep"); + break; + + case SVE_PRFOP_PLDL2STRM: + printf("pldl2strm"); + break; + + case SVE_PRFOP_PLDL3KEEP: + printf("pldl3keep"); + break; + + case SVE_PRFOP_PLDL3STRM: + printf("pldl3strm"); + break; + + case SVE_PRFOP_PSTL1KEEP: + printf("pstl1keep"); + break; + + case SVE_PRFOP_PSTL1STRM: + printf("pstl1strm"); + break; + + case SVE_PRFOP_PSTL2KEEP: + printf("pstl2keep"); + break; + + case SVE_PRFOP_PSTL2STRM: + printf("pstl2strm"); + break; + + case SVE_PRFOP_PSTL3KEEP: + printf("pstl3keep"); + break; + + case SVE_PRFOP_PSTL3STRM: + printf("pstl3strm"); + break; + + case SVE_PRFOP_CONST6: + printf("#6"); + break; + + case SVE_PRFOP_CONST7: + printf("#7"); + break; + + case SVE_PRFOP_CONST14: + printf("#0xE"); + break; + + case SVE_PRFOP_CONST15: + printf("#0xF"); + break; + + default: + assert(!"Invalid prfop"); + break; + } + + if (addComma) + { + emitDispComma(); + } +} + +/***************************************************************************** + * + * Returns the encoding to set the vector length specifier (vl) for an Arm64 SVE instruction + */ + +/*static*/ emitter::code_t emitter::insEncodeVectorLengthSpecifier(instrDesc* id) +{ + assert(id != nullptr); + assert(insOptsScalableStandard(id->idInsOpt())); + + if (id->idVectorLength4x()) + { + switch (id->idInsFmt()) + { + case IF_SVE_DL_2A: + return 0x400; // set the bit at location 10 + case IF_SVE_DY_3A: + return 0x2000; // set the bit at location 13 + default: + assert(!"Unexpected format"); + break; + } + } + + return 0; +} + #endif // TARGET_ARM64 From 60ae6f51e494624cbb1f69408b6d04fd002c573b Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:24:07 -0400 Subject: [PATCH 2/9] Remove emitDispLSExtendOpts (unused) --- src/coreclr/jit/emitarm64.cpp | 21 --------------------- src/coreclr/jit/emitarm64.h | 1 - 2 files changed, 22 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index a17a5990b7495..84fe569680a66 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -16828,27 +16828,6 @@ void emitter::emitDispExtendOpts(insOpts opt) assert(!"Bad value"); } -/***************************************************************************** - * - * Prints the encoding for the Extend Type encoding in loads/stores - */ - -void emitter::emitDispLSExtendOpts(insOpts opt) -{ - if (opt == INS_OPTS_LSL) - printf("LSL"); - else if (opt == INS_OPTS_UXTW) - printf("UXTW"); - else if (opt == INS_OPTS_UXTX) - printf("UXTX"); - else if (opt == INS_OPTS_SXTW) - printf("SXTW"); - else if (opt == INS_OPTS_SXTX) - printf("SXTX"); - else - assert(!"Bad value"); -} - //------------------------------------------------------------------------ // emitDispReg: Display a general-purpose register name or SIMD and floating-point scalar register name // diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 08e0f14a766cb..4576069c29722 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -56,7 +56,6 @@ void emitDispSveModAddr(instruction ins, regNumber reg1, regNumber reg2, insOpts void emitDispSveImm(regNumber reg1, ssize_t imm, insOpts opt); void emitDispSveImmMulVl(regNumber reg1, ssize_t imm); void emitDispSveImmIndex(regNumber reg1, insOpts opt, ssize_t imm); -void emitDispLSExtendOpts(insOpts opt); void emitDispReg(regNumber reg, emitAttr attr, bool addComma); void emitDispSveReg(regNumber reg, bool addComma); void emitDispSveReg(regNumber reg, insOpts opt, bool addComma); From 3a596b9ef8736c90964400b53b98c68a4c696b82 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:47:23 -0400 Subject: [PATCH 3/9] Move pred reg methods --- src/coreclr/jit/emitarm64.cpp | 581 +------------------------------ src/coreclr/jit/emitarm64.h | 2 +- src/coreclr/jit/emitarm64sve.cpp | 578 ++++++++++++++++++++++++++++++ 3 files changed, 580 insertions(+), 581 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 84fe569680a66..726f2e78fe3d8 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -3310,18 +3310,6 @@ static const char * const wRegNames[] = #include "register.h" }; - -static const char * const zRegNames[] = -{ - "z0", "z1", "z2", "z3", "z4", - "z5", "z6", "z7", "z8", "z9", - "z10", "z11", "z12", "z13", "z14", - "z15", "z16", "z17", "z18", "z19", - "z20", "z21", "z22", "z23", "z24", - "z25", "z26", "z27", "z28", "z29", - "z30", "z31" -}; - static const char * const vRegNames[] = { "v0", "v1", "v2", "v3", "v4", @@ -3365,22 +3353,6 @@ static const char * const bRegNames[] = "b30", "b31" }; -static const char * const pRegNames[] = -{ - "p0", "p1", "p2", "p3", "p4", - "p5", "p6", "p7", "p8", "p9", - "p10", "p11", "p12", "p13", "p14", - "p15" -}; - -static const char * const pnRegNames[] = -{ - "pn0", "pn1", "pn2", "pn3", "pn4", - "pn5", "pn6", "pn7", "pn8", "pn9", - "pn10", "pn11", "pn12", "pn13", "pn14", - "pn15" -}; - // clang-format on //------------------------------------------------------------------------ @@ -3424,7 +3396,7 @@ const char* emitter::emitRegName(regNumber reg, emitAttr size, bool varName) con } else if (size == EA_SCALABLE) { - rn = zRegNames[reg - REG_V0]; + rn = emitSveRegName(reg); } } @@ -3433,24 +3405,6 @@ const char* emitter::emitRegName(regNumber reg, emitAttr size, bool varName) con return rn; } -//------------------------------------------------------------------------ -// emitSveRegName: Returns a scalable vector register name. -// -// Arguments: -// reg - A SIMD and floating-point register. -// -// Return value: -// A string that represents a scalable vector register name. -// -const char* emitter::emitSveRegName(regNumber reg) -{ - assert((reg >= REG_V0) && (reg <= REG_V31)); - - int index = (int)reg - (int)REG_V0; - - return zRegNames[index]; -} - //------------------------------------------------------------------------ // emitVectorRegName: Returns a SIMD vector register name. // @@ -3469,25 +3423,6 @@ const char* emitter::emitVectorRegName(regNumber reg) return vRegNames[index]; } -//------------------------------------------------------------------------ -// emitPredicateRegName: Returns a predicate register name. -// -// Arguments: -// reg - A predicate register. -// -// Return value: -// A string that represents a predicate register name. -// -const char* emitter::emitPredicateRegName(regNumber reg, PredicateType ptype) -{ - assert((reg >= REG_P0) && (reg <= REG_P15)); - - const int index = (int)reg - (int)REG_P0; - const bool usePnRegs = (ptype == PREDICATE_N) || (ptype == PREDICATE_N_SIZED); - - return usePnRegs ? pnRegNames[index] : pRegNames[index]; -} - /***************************************************************************** * * Returns the base encoding of the given CPU instruction. @@ -6783,236 +6718,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return result; } -/************************************************************************ - * - * Convert a rotation value that is 90 or 270 into a smaller encoding that matches one-to-one with the 'rot' field. - */ - -/*static*/ ssize_t emitter::emitEncodeRotationImm90_or_270(ssize_t imm) -{ - switch (imm) - { - case 90: - return 0; - - case 270: - return 1; - - default: - break; - } - - assert(!"Invalid rotation value"); - return 0; -} - -/************************************************************************ - * - * Convert an encoded rotation value to 90 or 270. - */ - -/*static*/ ssize_t emitter::emitDecodeRotationImm90_or_270(ssize_t imm) -{ - assert(emitIsValidEncodedRotationImm0_to_270(imm)); - switch (imm) - { - case 0: - return 90; - - case 1: - return 270; - - default: - break; - } - - return 0; -} - -/************************************************************************ - * - * Check if the immediate value is a valid encoded rotation value for 90 or 270. - */ - -/*static*/ bool emitter::emitIsValidEncodedRotationImm90_or_270(ssize_t imm) -{ - return (imm == 0) || (imm == 1); -} - -/************************************************************************ - * - * Convert a rotation value that is 0, 90, 180 or 270 into a smaller encoding that matches one-to-one with the 'rot' - * field. - */ - -/*static*/ ssize_t emitter::emitEncodeRotationImm0_to_270(ssize_t imm) -{ - switch (imm) - { - case 0: - return 0; - - case 90: - return 1; - - case 180: - return 2; - - case 270: - return 3; - - default: - break; - } - - assert(!"Invalid rotation value"); - return 0; -} - -/************************************************************************ - * - * Convert an encoded rotation value to 0, 90, 180 or 270. - */ - -/*static*/ ssize_t emitter::emitDecodeRotationImm0_to_270(ssize_t imm) -{ - assert(emitIsValidEncodedRotationImm0_to_270(imm)); - switch (imm) - { - case 0: - return 0; - - case 1: - return 90; - - case 2: - return 180; - - case 3: - return 270; - - default: - break; - } - - return 0; -} - -/************************************************************************ - * - * Check if the immediate value is a valid encoded rotation value for 0, 90, 180 or 270. - */ - -/*static*/ bool emitter::emitIsValidEncodedRotationImm0_to_270(ssize_t imm) -{ - return (imm >= 0) && (imm <= 3); -} - -/************************************************************************ - * - * Convert a small immediate float value to an encoded version that matches one-to-one with the instructions. - * The instruction determines the value. - */ - -/*static*/ ssize_t emitter::emitEncodeSmallFloatImm(double immDbl, instruction ins) -{ -#ifdef DEBUG - switch (ins) - { - case INS_sve_fadd: - case INS_sve_fsub: - case INS_sve_fsubr: - assert((immDbl == 0.5) || (immDbl == 1.0)); - break; - - case INS_sve_fmax: - case INS_sve_fmaxnm: - case INS_sve_fmin: - case INS_sve_fminnm: - assert((immDbl == 0) || (immDbl == 1.0)); - break; - - case INS_sve_fmul: - assert((immDbl == 0.5) || (immDbl == 2.0)); - break; - - default: - assert(!"Invalid instruction"); - break; - } -#endif // DEBUG - if (immDbl < 1.0) - { - return 0; - } - return 1; -} - -/************************************************************************ - * - * Convert an encoded small float immediate value. The instruction determines the value. - */ - -/*static*/ double emitter::emitDecodeSmallFloatImm(ssize_t imm, instruction ins) -{ - assert(emitIsValidEncodedSmallFloatImm(imm)); - switch (ins) - { - case INS_sve_fadd: - case INS_sve_fsub: - case INS_sve_fsubr: - if (imm == 0) - { - return 0.5; - } - else - { - return 1.0; - } - - case INS_sve_fmax: - case INS_sve_fmaxnm: - case INS_sve_fmin: - case INS_sve_fminnm: - if (imm == 0) - { - return 0.0; - } - else - { - return 1.0; - } - break; - - case INS_sve_fmul: - if (imm == 0) - { - return 0.5; - } - else - { - return 2.0; - } - break; - - default: - break; - } - - assert(!"Invalid instruction"); - return 0.0; -} - -/************************************************************************ - * - * Check if the immediate value is a valid encoded small float. - */ - -/*static*/ bool emitter::emitIsValidEncodedSmallFloatImm(size_t imm) -{ - return (imm == 0) || (imm == 1); -} - /***************************************************************************** * * For the given 'ins' returns the reverse instruction @@ -7190,65 +6895,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return registerListSize; } -/***************************************************************************** - * - * Expands an option that has different size operands (INS_OPTS_*_TO_*) into - * a pair of scalable options where the first describes the size of the - * destination operand and the second describes the size of the source operand. - */ - -/*static*/ void emitter::optExpandConversionPair(insOpts opt, insOpts& dst, insOpts& src) -{ - dst = INS_OPTS_NONE; - src = INS_OPTS_NONE; - - switch (opt) - { - case INS_OPTS_H_TO_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_S_TO_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_S_TO_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_D_TO_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_D; - break; - case INS_OPTS_H_TO_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_D_TO_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_D; - break; - case INS_OPTS_SCALABLE_H: - dst = INS_OPTS_SCALABLE_H; - src = INS_OPTS_SCALABLE_H; - break; - case INS_OPTS_SCALABLE_S: - dst = INS_OPTS_SCALABLE_S; - src = INS_OPTS_SCALABLE_S; - break; - case INS_OPTS_SCALABLE_D: - dst = INS_OPTS_SCALABLE_D; - src = INS_OPTS_SCALABLE_D; - break; - default: - noway_assert(!"unreachable"); - break; - } - - assert(dst != INS_OPTS_NONE && src != INS_OPTS_NONE); - return; -} - // For the given 'arrangement' returns the 'datasize' specified by the vector register arrangement // asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed // @@ -7299,66 +6945,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) } } -// For the given 'elemsize' returns the 'arrangement' when used in a SVE vector register arrangement. -// Asserts and returns INS_OPTS_NONE if an invalid 'elemsize' is passed -// -/*static*/ insOpts emitter::optGetSveInsOpt(emitAttr elemsize) -{ - switch (elemsize) - { - case EA_1BYTE: - return INS_OPTS_SCALABLE_B; - - case EA_2BYTE: - return INS_OPTS_SCALABLE_H; - - case EA_4BYTE: - return INS_OPTS_SCALABLE_S; - - case EA_8BYTE: - return INS_OPTS_SCALABLE_D; - - case EA_16BYTE: - return INS_OPTS_SCALABLE_Q; - - default: - assert(!"Invalid emitAttr for sve vector register"); - return INS_OPTS_NONE; - } -} - -// For the given 'arrangement' returns the 'elemsize' specified by the SVE vector register arrangement -// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed -// -/*static*/ emitAttr emitter::optGetSveElemsize(insOpts arrangement) -{ - switch (arrangement) - { - case INS_OPTS_SCALABLE_B: - return EA_1BYTE; - - case INS_OPTS_SCALABLE_H: - return EA_2BYTE; - - case INS_OPTS_SCALABLE_S: - case INS_OPTS_SCALABLE_S_UXTW: - case INS_OPTS_SCALABLE_S_SXTW: - return EA_4BYTE; - - case INS_OPTS_SCALABLE_D: - case INS_OPTS_SCALABLE_D_UXTW: - case INS_OPTS_SCALABLE_D_SXTW: - return EA_8BYTE; - - case INS_OPTS_SCALABLE_Q: - return EA_16BYTE; - - default: - assert(!"Invalid insOpt for vector register"); - return EA_UNKNOWN; - } -} - /*static*/ insOpts emitter::optWidenElemsizeArrangement(insOpts arrangement) { if ((arrangement == INS_OPTS_8B) || (arrangement == INS_OPTS_16B)) @@ -7380,47 +6966,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) } } -/*static*/ insOpts emitter::optWidenSveElemsizeArrangement(insOpts arrangement) -{ - switch (arrangement) - { - case INS_OPTS_SCALABLE_B: - return INS_OPTS_SCALABLE_H; - - case INS_OPTS_SCALABLE_H: - return INS_OPTS_SCALABLE_S; - - case INS_OPTS_SCALABLE_S: - return INS_OPTS_SCALABLE_D; - - default: - assert(!" invalid 'arrangement' value"); - return INS_OPTS_NONE; - } -} - -/*static*/ insOpts emitter::optSveToQuadwordElemsizeArrangement(insOpts arrangement) -{ - switch (arrangement) - { - case INS_OPTS_SCALABLE_B: - return INS_OPTS_16B; - - case INS_OPTS_SCALABLE_H: - return INS_OPTS_8H; - - case INS_OPTS_SCALABLE_S: - return INS_OPTS_4S; - - case INS_OPTS_SCALABLE_D: - return INS_OPTS_2D; - - default: - assert(!" invalid 'arrangement' value"); - return INS_OPTS_NONE; - } -} - /*static*/ emitAttr emitter::widenDatasize(emitAttr datasize) { if (datasize == EA_1BYTE) @@ -13199,19 +12744,6 @@ void emitter::emitIns_Call(EmitCallType callType, return ureg << 5; } -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '12' thru '10' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_12_to_10(regNumber reg) -{ - assert(isLowPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 7)); - return ureg << 10; -} - /***************************************************************************** * * Return an encoding for the specified 'V' register used in '20' thru '16' position. @@ -13264,58 +12796,6 @@ void emitter::emitIns_Call(EmitCallType callType, return ureg << 0; } -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '19' thru '16' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_19_to_16(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 16; -} - -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '3' thru '0' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_0(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg; -} - -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '8' thru '5' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_8_to_5(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 5; -} - -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '13' thru '10' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_13_to_10(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 10; -} - /***************************************************************************** * * Return an encoding for the specified 'R' register used in '17' thru '16' position. @@ -13329,65 +12809,6 @@ void emitter::emitIns_Call(EmitCallType callType, return ureg << 16; } -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '7' thru '5' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_7_to_5(regNumber reg) -{ - assert(isHighPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P8; - assert((ureg >= 0) && (ureg <= 7)); - return ureg << 5; -} - -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '3' thru '1' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_1(regNumber reg) -{ - assert(isLowPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 0) && (ureg <= 15)); - return ureg << 1; -} - -/***************************************************************************** - * - * Return an encoding for the specified 'P' register used in '2' thru '0' position. - */ - -/*static*/ emitter::code_t emitter::insEncodeReg_P_2_to_0(regNumber reg) -{ - assert(isPredicateRegister(reg)); - emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; - assert((ureg >= 8) && (ureg <= 15)); - return (ureg - 8) << 0; -} - -/***************************************************************************** - * - * Return an encoding for the specified predicate type used in '16' position. - */ - -/*static*/ emitter::code_t emitter::insEncodePredQualifier_16(bool merge) -{ - return merge ? 1 << 16 : 0; -} - -/***************************************************************************** - * - * Return an encoding for the specified predicate type used in '4' position. - */ - -/*static*/ emitter::code_t emitter::insEncodePredQualifier_4(bool merge) -{ - return merge ? 1 << 4 : 0; -} - /***************************************************************************** * * Return an encoding for the specified 'V' register used in '18' thru '16' position. diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 4576069c29722..7b3a7bf9793ca 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -29,7 +29,7 @@ enum PredicateType PREDICATE_N_SIZED, // Predicate printed as counter with element size }; -const char* emitSveRegName(regNumber reg); +const char* emitSveRegName(regNumber reg) const; const char* emitVectorRegName(regNumber reg); const char* emitPredicateRegName(regNumber reg, PredicateType ptype); diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index d19528dd8f8cc..99d7d3f9a78be 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -25,6 +25,33 @@ XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX /*****************************************************************************/ // clang-format off +static const char * const zRegNames[] = +{ + "z0", "z1", "z2", "z3", "z4", + "z5", "z6", "z7", "z8", "z9", + "z10", "z11", "z12", "z13", "z14", + "z15", "z16", "z17", "z18", "z19", + "z20", "z21", "z22", "z23", "z24", + "z25", "z26", "z27", "z28", "z29", + "z30", "z31" +}; + +static const char * const pRegNames[] = +{ + "p0", "p1", "p2", "p3", "p4", + "p5", "p6", "p7", "p8", "p9", + "p10", "p11", "p12", "p13", "p14", + "p15" +}; + +static const char * const pnRegNames[] = +{ + "pn0", "pn1", "pn2", "pn3", "pn4", + "pn5", "pn6", "pn7", "pn8", "pn9", + "pn10", "pn11", "pn12", "pn13", "pn14", + "pn15" +}; + static const char * const svePatternNames[] = { "pow2", "vl1", "vl2", "vl3", @@ -8165,6 +8192,236 @@ void emitter::emitIns_PRFOP_R_R_I(instruction ins, assert(isValidSimm<5>(*imm2)); } +/************************************************************************ + * + * Convert a small immediate float value to an encoded version that matches one-to-one with the instructions. + * The instruction determines the value. + */ + +/*static*/ ssize_t emitter::emitEncodeSmallFloatImm(double immDbl, instruction ins) +{ +#ifdef DEBUG + switch (ins) + { + case INS_sve_fadd: + case INS_sve_fsub: + case INS_sve_fsubr: + assert((immDbl == 0.5) || (immDbl == 1.0)); + break; + + case INS_sve_fmax: + case INS_sve_fmaxnm: + case INS_sve_fmin: + case INS_sve_fminnm: + assert((immDbl == 0) || (immDbl == 1.0)); + break; + + case INS_sve_fmul: + assert((immDbl == 0.5) || (immDbl == 2.0)); + break; + + default: + assert(!"Invalid instruction"); + break; + } +#endif // DEBUG + if (immDbl < 1.0) + { + return 0; + } + return 1; +} + +/************************************************************************ + * + * Convert an encoded small float immediate value. The instruction determines the value. + */ + +/*static*/ double emitter::emitDecodeSmallFloatImm(ssize_t imm, instruction ins) +{ + assert(emitIsValidEncodedSmallFloatImm(imm)); + switch (ins) + { + case INS_sve_fadd: + case INS_sve_fsub: + case INS_sve_fsubr: + if (imm == 0) + { + return 0.5; + } + else + { + return 1.0; + } + + case INS_sve_fmax: + case INS_sve_fmaxnm: + case INS_sve_fmin: + case INS_sve_fminnm: + if (imm == 0) + { + return 0.0; + } + else + { + return 1.0; + } + break; + + case INS_sve_fmul: + if (imm == 0) + { + return 0.5; + } + else + { + return 2.0; + } + break; + + default: + break; + } + + assert(!"Invalid instruction"); + return 0.0; +} + +/************************************************************************ + * + * Check if the immediate value is a valid encoded small float. + */ + +/*static*/ bool emitter::emitIsValidEncodedSmallFloatImm(size_t imm) +{ + return (imm == 0) || (imm == 1); +} + +/************************************************************************ + * + * Convert a rotation value that is 90 or 270 into a smaller encoding that matches one-to-one with the 'rot' field. + */ + +/*static*/ ssize_t emitter::emitEncodeRotationImm90_or_270(ssize_t imm) +{ + switch (imm) + { + case 90: + return 0; + + case 270: + return 1; + + default: + break; + } + + assert(!"Invalid rotation value"); + return 0; +} + +/************************************************************************ + * + * Convert an encoded rotation value to 90 or 270. + */ + +/*static*/ ssize_t emitter::emitDecodeRotationImm90_or_270(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm0_to_270(imm)); + switch (imm) + { + case 0: + return 90; + + case 1: + return 270; + + default: + break; + } + + return 0; +} + +/************************************************************************ + * + * Check if the immediate value is a valid encoded rotation value for 90 or 270. + */ + +/*static*/ bool emitter::emitIsValidEncodedRotationImm90_or_270(ssize_t imm) +{ + return (imm == 0) || (imm == 1); +} + +/************************************************************************ + * + * Convert a rotation value that is 0, 90, 180 or 270 into a smaller encoding that matches one-to-one with the 'rot' + * field. + */ + +/*static*/ ssize_t emitter::emitEncodeRotationImm0_to_270(ssize_t imm) +{ + switch (imm) + { + case 0: + return 0; + + case 90: + return 1; + + case 180: + return 2; + + case 270: + return 3; + + default: + break; + } + + assert(!"Invalid rotation value"); + return 0; +} + +/************************************************************************ + * + * Convert an encoded rotation value to 0, 90, 180 or 270. + */ + +/*static*/ ssize_t emitter::emitDecodeRotationImm0_to_270(ssize_t imm) +{ + assert(emitIsValidEncodedRotationImm0_to_270(imm)); + switch (imm) + { + case 0: + return 0; + + case 1: + return 90; + + case 2: + return 180; + + case 3: + return 270; + + default: + break; + } + + return 0; +} + +/************************************************************************ + * + * Check if the immediate value is a valid encoded rotation value for 0, 90, 180 or 270. + */ + +/*static*/ bool emitter::emitIsValidEncodedRotationImm0_to_270(ssize_t imm) +{ + return (imm >= 0) && (imm <= 3); +} + /***************************************************************************** * * Returns the encoding to select an insSvePattern @@ -10410,6 +10667,43 @@ void emitter::emitDispSveConsecutiveRegList(regNumber firstReg, unsigned listSiz } } +//------------------------------------------------------------------------ +// emitSveRegName: Returns a scalable vector register name. +// +// Arguments: +// reg - A SIMD and floating-point register. +// +// Return value: +// A string that represents a scalable vector register name. +// +const char* emitter::emitSveRegName(regNumber reg) const +{ + assert((reg >= REG_V0) && (reg <= REG_V31)); + + int index = (int)reg - (int)REG_V0; + + return zRegNames[index]; +} + +//------------------------------------------------------------------------ +// emitPredicateRegName: Returns a predicate register name. +// +// Arguments: +// reg - A predicate register. +// +// Return value: +// A string that represents a predicate register name. +// +const char* emitter::emitPredicateRegName(regNumber reg, PredicateType ptype) +{ + assert((reg >= REG_P0) && (reg <= REG_P15)); + + const int index = (int)reg - (int)REG_P0; + const bool usePnRegs = (ptype == PREDICATE_N) || (ptype == PREDICATE_N_SIZED); + + return usePnRegs ? pnRegNames[index] : pRegNames[index]; +} + //------------------------------------------------------------------------ // emitDispPredicateReg: Display a predicate register name with with an arrangement suffix // @@ -10613,4 +10907,288 @@ void emitter::emitDispSvePrfop(insSvePrfop prfop, bool addComma) return 0; } +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '19' thru '16' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_19_to_16(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 16; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '3' thru '0' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_0(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '8' thru '5' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_8_to_5(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 5; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '13' thru '10' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_13_to_10(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 10; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '12' thru '10' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_12_to_10(regNumber reg) +{ + assert(isLowPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 7)); + return ureg << 10; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '7' thru '5' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_7_to_5(regNumber reg) +{ + assert(isHighPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P8; + assert((ureg >= 0) && (ureg <= 7)); + return ureg << 5; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '3' thru '1' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_3_to_1(regNumber reg) +{ + assert(isLowPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 0) && (ureg <= 15)); + return ureg << 1; +} + +/***************************************************************************** + * + * Return an encoding for the specified 'P' register used in '2' thru '0' position. + */ + +/*static*/ emitter::code_t emitter::insEncodeReg_P_2_to_0(regNumber reg) +{ + assert(isPredicateRegister(reg)); + emitter::code_t ureg = (emitter::code_t)reg - (emitter::code_t)REG_P0; + assert((ureg >= 8) && (ureg <= 15)); + return (ureg - 8) << 0; +} + +/***************************************************************************** + * + * Return an encoding for the specified predicate type used in '16' position. + */ + +/*static*/ emitter::code_t emitter::insEncodePredQualifier_16(bool merge) +{ + return merge ? 1 << 16 : 0; +} + +/***************************************************************************** + * + * Return an encoding for the specified predicate type used in '4' position. + */ + +/*static*/ emitter::code_t emitter::insEncodePredQualifier_4(bool merge) +{ + return merge ? 1 << 4 : 0; +} + +// For the given 'elemsize' returns the 'arrangement' when used in a SVE vector register arrangement. +// Asserts and returns INS_OPTS_NONE if an invalid 'elemsize' is passed +// +/*static*/ insOpts emitter::optGetSveInsOpt(emitAttr elemsize) +{ + switch (elemsize) + { + case EA_1BYTE: + return INS_OPTS_SCALABLE_B; + + case EA_2BYTE: + return INS_OPTS_SCALABLE_H; + + case EA_4BYTE: + return INS_OPTS_SCALABLE_S; + + case EA_8BYTE: + return INS_OPTS_SCALABLE_D; + + case EA_16BYTE: + return INS_OPTS_SCALABLE_Q; + + default: + assert(!"Invalid emitAttr for sve vector register"); + return INS_OPTS_NONE; + } +} + +// For the given 'arrangement' returns the 'elemsize' specified by the SVE vector register arrangement +// asserts and returns EA_UNKNOWN if an invalid 'arrangement' value is passed +// +/*static*/ emitAttr emitter::optGetSveElemsize(insOpts arrangement) +{ + switch (arrangement) + { + case INS_OPTS_SCALABLE_B: + return EA_1BYTE; + + case INS_OPTS_SCALABLE_H: + return EA_2BYTE; + + case INS_OPTS_SCALABLE_S: + case INS_OPTS_SCALABLE_S_UXTW: + case INS_OPTS_SCALABLE_S_SXTW: + return EA_4BYTE; + + case INS_OPTS_SCALABLE_D: + case INS_OPTS_SCALABLE_D_UXTW: + case INS_OPTS_SCALABLE_D_SXTW: + return EA_8BYTE; + + case INS_OPTS_SCALABLE_Q: + return EA_16BYTE; + + default: + assert(!"Invalid insOpt for vector register"); + return EA_UNKNOWN; + } +} + +/*static*/ insOpts emitter::optWidenSveElemsizeArrangement(insOpts arrangement) +{ + switch (arrangement) + { + case INS_OPTS_SCALABLE_B: + return INS_OPTS_SCALABLE_H; + + case INS_OPTS_SCALABLE_H: + return INS_OPTS_SCALABLE_S; + + case INS_OPTS_SCALABLE_S: + return INS_OPTS_SCALABLE_D; + + default: + assert(!" invalid 'arrangement' value"); + return INS_OPTS_NONE; + } +} + +/*static*/ insOpts emitter::optSveToQuadwordElemsizeArrangement(insOpts arrangement) +{ + switch (arrangement) + { + case INS_OPTS_SCALABLE_B: + return INS_OPTS_16B; + + case INS_OPTS_SCALABLE_H: + return INS_OPTS_8H; + + case INS_OPTS_SCALABLE_S: + return INS_OPTS_4S; + + case INS_OPTS_SCALABLE_D: + return INS_OPTS_2D; + + default: + assert(!" invalid 'arrangement' value"); + return INS_OPTS_NONE; + } +} + +/***************************************************************************** + * + * Expands an option that has different size operands (INS_OPTS_*_TO_*) into + * a pair of scalable options where the first describes the size of the + * destination operand and the second describes the size of the source operand. + */ + +/*static*/ void emitter::optExpandConversionPair(insOpts opt, insOpts& dst, insOpts& src) +{ + dst = INS_OPTS_NONE; + src = INS_OPTS_NONE; + + switch (opt) + { + case INS_OPTS_H_TO_S: + dst = INS_OPTS_SCALABLE_S; + src = INS_OPTS_SCALABLE_H; + break; + case INS_OPTS_S_TO_H: + dst = INS_OPTS_SCALABLE_H; + src = INS_OPTS_SCALABLE_S; + break; + case INS_OPTS_S_TO_D: + dst = INS_OPTS_SCALABLE_D; + src = INS_OPTS_SCALABLE_S; + break; + case INS_OPTS_D_TO_S: + dst = INS_OPTS_SCALABLE_S; + src = INS_OPTS_SCALABLE_D; + break; + case INS_OPTS_H_TO_D: + dst = INS_OPTS_SCALABLE_D; + src = INS_OPTS_SCALABLE_H; + break; + case INS_OPTS_D_TO_H: + dst = INS_OPTS_SCALABLE_H; + src = INS_OPTS_SCALABLE_D; + break; + case INS_OPTS_SCALABLE_H: + dst = INS_OPTS_SCALABLE_H; + src = INS_OPTS_SCALABLE_H; + break; + case INS_OPTS_SCALABLE_S: + dst = INS_OPTS_SCALABLE_S; + src = INS_OPTS_SCALABLE_S; + break; + case INS_OPTS_SCALABLE_D: + dst = INS_OPTS_SCALABLE_D; + src = INS_OPTS_SCALABLE_D; + break; + default: + noway_assert(!"unreachable"); + break; + } + + assert(dst != INS_OPTS_NONE && src != INS_OPTS_NONE); + return; +} + #endif // TARGET_ARM64 From e23e449ac0cbc0961f49785622049ecf54af77e2 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:49:08 -0400 Subject: [PATCH 4/9] Remove emitEncodeFloatImm8 (unused) --- src/coreclr/jit/emitarm64.cpp | 16 ---------------- src/coreclr/jit/emitarm64.h | 2 -- 2 files changed, 18 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 726f2e78fe3d8..1e9ea9aabfdb7 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -6702,22 +6702,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return canEncode; } -/************************************************************************ - * - * Convert a double into its 'float 8-bit immediate' representation - */ - -/*static*/ emitter::floatImm8 emitter::emitEncodeFloatImm8(double immDbl) -{ - emitter::floatImm8 result; - result.immFPIVal = 0; - - bool canEncode = canEncodeFloatImm8(immDbl, &result); - assert(canEncode); - - return result; -} - /***************************************************************************** * * For the given 'ins' returns the reverse instruction diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 7b3a7bf9793ca..be2aa7c3de6d2 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -295,8 +295,6 @@ union floatImm8 { * Convert between a double and its 'float 8-bit immediate' representation */ -static emitter::floatImm8 emitEncodeFloatImm8(double immDbl); - static double emitDecodeFloatImm8(const emitter::floatImm8 fpImm); static ssize_t emitEncodeRotationImm90_or_270(ssize_t imm); From d33849922eba4c8ae2d71a3a17d48e999931631e Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:51:36 -0400 Subject: [PATCH 5/9] Remove emitEncodeByteShiftedImm (unused) --- src/coreclr/jit/emitarm64.cpp | 16 ---------------- src/coreclr/jit/emitarm64.h | 2 -- 2 files changed, 18 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 1e9ea9aabfdb7..89d8a723e81d3 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -6601,22 +6601,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return false; } -/************************************************************************ - * - * Convert a 32-bit immediate into its 'byteShifted immediate' representation imm(i8,by) - */ - -/*static*/ emitter::byteShiftedImm emitter::emitEncodeByteShiftedImm(INT64 imm, emitAttr size, bool allow_MSL) -{ - emitter::byteShiftedImm result; - result.immBSVal = 0; - - bool canEncode = canEncodeByteShiftedImm(imm, size, allow_MSL, &result); - assert(canEncode); - - return result; -} - /************************************************************************ * * Convert a 'float 8-bit immediate' into a double. diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index be2aa7c3de6d2..d379988e0d449 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -270,8 +270,6 @@ union byteShiftedImm { * representation imm(i8,by) */ -static emitter::byteShiftedImm emitEncodeByteShiftedImm(INT64 imm, emitAttr size, bool allow_MSL); - static UINT32 emitDecodeByteShiftedImm(const emitter::byteShiftedImm bsImm, emitAttr size); /************************************************************************ From 540005cdca01f10a503d11fb5e9149e69e9287bf Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:52:37 -0400 Subject: [PATCH 6/9] Remove emitEncodeHalfwordImm (unused) --- src/coreclr/jit/emitarm64.cpp | 16 ---------------- src/coreclr/jit/emitarm64.h | 2 -- 2 files changed, 18 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 89d8a723e81d3..acfbf48915e59 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -6445,22 +6445,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return false; } -/************************************************************************ - * - * Convert a 64-bit immediate into its 'halfword immediate' representation imm(i16,hw) - */ - -/*static*/ emitter::halfwordImm emitter::emitEncodeHalfwordImm(INT64 imm, emitAttr size) -{ - emitter::halfwordImm result; - result.immHWVal = 0; - - bool canEncode = canEncodeHalfwordImm(imm, size, &result); - assert(canEncode); - - return result; -} - /************************************************************************ * * Convert an imm(i8,sh) into a 16/32-bit immediate diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index d379988e0d449..9ac5848593f57 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -244,8 +244,6 @@ union halfwordImm { * representation imm(i16,hw) */ -static emitter::halfwordImm emitEncodeHalfwordImm(INT64 imm, emitAttr size); - static INT64 emitDecodeHalfwordImm(const emitter::halfwordImm hwImm, emitAttr size); /************************************************************************ From 50b1eeb939edf242f750d37c64d07fff2db6c8ef Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:54:01 -0400 Subject: [PATCH 7/9] Remove emitEncodeBitMaskImm (unused) --- src/coreclr/jit/emitarm64.cpp | 16 ---------------- src/coreclr/jit/emitarm64.h | 2 -- 2 files changed, 18 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index acfbf48915e59..cb28042000caa 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -6354,22 +6354,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return false; } -/************************************************************************ - * - * Convert a 64-bit immediate into its 'bitmask immediate' representation imm(N,r,s) - */ - -/*static*/ emitter::bitMaskImm emitter::emitEncodeBitMaskImm(INT64 imm, emitAttr size) -{ - emitter::bitMaskImm result; - result.immNRS = 0; - - bool canEncode = canEncodeBitMaskImm(imm, size, &result); - assert(canEncode); - - return result; -} - /************************************************************************ * * Convert an imm(i16,hw) into a 32/64-bit immediate diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 9ac5848593f57..6b886ad757ac7 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -219,8 +219,6 @@ union bitMaskImm { * representation imm(i16,hw) */ -static emitter::bitMaskImm emitEncodeBitMaskImm(INT64 imm, emitAttr size); - static INT64 emitDecodeBitMaskImm(const emitter::bitMaskImm bmImm, emitAttr size); /************************************************************************ From 5ab05d7e59a58fbfb85f11987ea843e689c573b8 Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 14:56:35 -0400 Subject: [PATCH 8/9] Remove normalizeImm32 (unused) --- src/coreclr/jit/emitarm64.cpp | 26 -------------------------- src/coreclr/jit/emitarm64.h | 3 --- 2 files changed, 29 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index cb28042000caa..1d5aca2f5868c 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -6104,32 +6104,6 @@ emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) return result; } -/***************************************************************************** - * - * Normalize the 'imm' so that the upper bits, as defined by 'size' are zero - */ - -/*static*/ INT32 emitter::normalizeImm32(INT32 imm, emitAttr size) -{ - unsigned immWidth = getBitWidth(size); - INT32 result = imm; - - if (immWidth < 32) - { - // Check that 'imm' fits in 'immWidth' bits. Don't consider "sign" bits above width. - INT32 maxVal = 1 << immWidth; - INT32 lowBitsMask = maxVal - 1; - INT32 hiBitsMask = ~lowBitsMask; - INT32 signBitsMask = hiBitsMask | (1 << (immWidth - 1)); // The high bits must be set, and the top bit - // (sign bit) must be set. - assert((imm < maxVal) || ((imm & signBitsMask) == signBitsMask)); - - // mask off the hiBits - result &= lowBitsMask; - } - return result; -} - /************************************************************************ * * returns true if 'imm' of 'size bits (8/16/32/64) can be encoded diff --git a/src/coreclr/jit/emitarm64.h b/src/coreclr/jit/emitarm64.h index 6b886ad757ac7..0ddf5a86c803c 100644 --- a/src/coreclr/jit/emitarm64.h +++ b/src/coreclr/jit/emitarm64.h @@ -1045,9 +1045,6 @@ static bool canEncodeWithShiftImmBy12(INT64 imm); // Normalize the 'imm' so that the upper bits, as defined by 'size' are zero static INT64 normalizeImm64(INT64 imm, emitAttr size); -// Normalize the 'imm' so that the upper bits, as defined by 'size' are zero -static INT32 normalizeImm32(INT32 imm, emitAttr size); - // true if 'imm' can be encoded using a 'bitmask immediate', also returns the encoding if wbBMI is non-null static bool canEncodeBitMaskImm(INT64 imm, emitAttr size, emitter::bitMaskImm* wbBMI = nullptr); From cbec6ea9f27ab4d78796503be454dba271038ebf Mon Sep 17 00:00:00 2001 From: "Aman Khalid (from Dev Box)" Date: Wed, 20 Mar 2024 15:16:45 -0400 Subject: [PATCH 9/9] Move emitInsCodeSve --- src/coreclr/jit/emitarm64.cpp | 1432 ------------------------------ src/coreclr/jit/emitarm64sve.cpp | 1432 ++++++++++++++++++++++++++++++ 2 files changed, 1432 insertions(+), 1432 deletions(-) diff --git a/src/coreclr/jit/emitarm64.cpp b/src/coreclr/jit/emitarm64.cpp index 1d5aca2f5868c..2ba16a9b39c69 100644 --- a/src/coreclr/jit/emitarm64.cpp +++ b/src/coreclr/jit/emitarm64.cpp @@ -4321,1438 +4321,6 @@ emitter::code_t emitter::emitInsCode(instruction ins, insFormat fmt) return code; } -/***************************************************************************** - * - * Returns the specific encoding of the given CPU instruction and format - */ - -emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) -{ - // clang-format off - const static code_t insCodes1[] = - { - #define INST1(id, nm, info, fmt, e1 ) e1, - #define INST2(id, nm, info, fmt, e1, e2 ) e1, - #define INST3(id, nm, info, fmt, e1, e2, e3 ) e1, - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e1, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e1, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e1, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e1, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e1, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e1, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e1, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e1, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes2[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) e2, - #define INST3(id, nm, info, fmt, e1, e2, e3 ) e2, - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e2, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e2, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e2, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e2, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e2, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e2, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e2, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e2, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes3[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) e3, - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e3, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e3, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e3, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e3, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e3, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e3, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e3, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e3, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes4[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e4, - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e4, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e4, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e4, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e4, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e4, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e4, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e4, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes5[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e5, - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e5, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e5, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e5, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e5, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e5, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e5, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes6[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e6, - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e6, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e6, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e6, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e6, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e6, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes7[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e7, - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e7, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e7, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e7, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e7, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes8[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e8, - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e8, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e8, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e8, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes9[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e9, - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e9, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e9, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes10[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e10, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e10, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes11[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e11, - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e11, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes12[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e12, - #include "instrsarm64sve.h" - }; - - const static code_t insCodes13[] = - { - #define INST1(id, nm, info, fmt, e1 ) - #define INST2(id, nm, info, fmt, e1, e2 ) - #define INST3(id, nm, info, fmt, e1, e2, e3 ) - #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) - #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) - #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) - #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) - #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) - #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) - #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) - #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e13, - #include "instrsarm64sve.h" - }; - - // clang-format on - const static insFormat formatEncode13A[13] = {IF_SVE_AU_3A, IF_SVE_BT_1A, IF_SVE_BV_2A, IF_SVE_BV_2A_J, - IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_CP_3A, IF_SVE_CQ_3A, - IF_SVE_CW_4A, IF_SVE_CZ_4A, IF_SVE_CZ_4A_K, IF_SVE_CZ_4A_L, - IF_SVE_EB_1A}; - const static insFormat formatEncode11A[11] = {IF_SVE_JD_4B, IF_SVE_JD_4C, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, - IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, IF_SVE_JJ_4B, - IF_SVE_JJ_4B_E, IF_SVE_JN_3B, IF_SVE_JN_3C}; - const static insFormat formatEncode9A[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, - IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, - IF_SVE_HX_3A_E, IF_SVE_IJ_3A_F, IF_SVE_IK_4A_G}; - const static insFormat formatEncode9B[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, - IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, - IF_SVE_HX_3A_E, IF_SVE_IJ_3A_G, IF_SVE_IK_4A_I}; - const static insFormat formatEncode9C[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, - IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, - IF_SVE_HX_3A_E, IF_SVE_IH_3A_F, IF_SVE_II_4A_H}; - const static insFormat formatEncode9D[9] = {IF_SVE_IH_3A, IF_SVE_IH_3A_A, IF_SVE_II_4A, - IF_SVE_II_4A_B, IF_SVE_IU_4A, IF_SVE_IU_4A_C, - IF_SVE_IU_4B, IF_SVE_IU_4B_D, IF_SVE_IV_3A}; - const static insFormat formatEncode9E[9] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, - IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, - IF_SVE_JJ_4B, IF_SVE_JJ_4B_E, IF_SVE_JN_3A}; - const static insFormat formatEncode9F[9] = {IF_SVE_JD_4C, IF_SVE_JD_4C_A, IF_SVE_JJ_4A, - IF_SVE_JJ_4A_B, IF_SVE_JJ_4B, IF_SVE_JJ_4B_C, - IF_SVE_JL_3A, IF_SVE_JN_3C, IF_SVE_JN_3C_D}; - const static insFormat formatEncode8A[8] = {IF_SVE_CE_2A, IF_SVE_CE_2B, IF_SVE_CE_2C, IF_SVE_CE_2D, - IF_SVE_CF_2A, IF_SVE_CF_2B, IF_SVE_CF_2C, IF_SVE_CF_2D}; - const static insFormat formatEncode8B[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, - IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_F}; - const static insFormat formatEncode8C[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, - IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_G}; - const static insFormat formatEncode7A[7] = {IF_SVE_IJ_3A, IF_SVE_IK_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, - IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; - const static insFormat formatEncode6A[6] = {IF_SVE_AE_3A, IF_SVE_BD_3A, IF_SVE_EE_1A, - IF_SVE_FD_3A, IF_SVE_FD_3B, IF_SVE_FD_3C}; - const static insFormat formatEncode6B[6] = {IF_SVE_GY_3A, IF_SVE_GY_3B, IF_SVE_GY_3B_D, - IF_SVE_HA_3A, IF_SVE_HA_3A_E, IF_SVE_HA_3A_F}; - const static insFormat formatEncode6C[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, - IF_SVE_HX_3A_B, IF_SVE_IJ_3A_D, IF_SVE_IK_4A_F}; - const static insFormat formatEncode6D[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, - IF_SVE_HX_3A_B, IF_SVE_IJ_3A_E, IF_SVE_IK_4A_H}; - const static insFormat formatEncode6E[6] = {IF_SVE_HY_3A, IF_SVE_HY_3A_A, IF_SVE_HY_3B, - IF_SVE_HZ_2A_B, IF_SVE_IA_2A, IF_SVE_IB_3A}; - const static insFormat formatEncode6F[6] = {IF_SVE_IG_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, - IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; - const static insFormat formatEncode6G[6] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JK_4A, - IF_SVE_JK_4A_B, IF_SVE_JK_4B, IF_SVE_JN_3A}; - const static insFormat formatEncode5A[5] = {IF_SVE_AM_2A, IF_SVE_AN_3A, IF_SVE_AO_3A, IF_SVE_BF_2A, IF_SVE_BG_3A}; - const static insFormat formatEncode5B[5] = {IF_SVE_GX_3A, IF_SVE_GX_3B, IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; - const static insFormat formatEncode5C[5] = {IF_SVE_EF_3A, IF_SVE_EG_3A, IF_SVE_EH_3A, IF_SVE_EY_3A, IF_SVE_EY_3B}; - const static insFormat formatEncode5D[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, - IF_SVE_IG_4A_D}; - const static insFormat formatEncode5E[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, - IF_SVE_IG_4A_E}; - const static insFormat formatEncode4A[4] = {IF_SVE_AA_3A, IF_SVE_AU_3A, IF_SVE_BS_1A, IF_SVE_CZ_4A}; - const static insFormat formatEncode4B[4] = {IF_SVE_BU_2A, IF_SVE_BV_2B, IF_SVE_EA_1A, IF_SVE_EB_1B}; - const static insFormat formatEncode4E[4] = {IF_SVE_BE_3A, IF_SVE_FI_3A, IF_SVE_FI_3B, IF_SVE_FI_3C}; - const static insFormat formatEncode4F[4] = {IF_SVE_EM_3A, IF_SVE_FK_3A, IF_SVE_FK_3B, IF_SVE_FK_3C}; - const static insFormat formatEncode4G[4] = {IF_SVE_AR_4A, IF_SVE_FF_3A, IF_SVE_FF_3B, IF_SVE_FF_3C}; - const static insFormat formatEncode4H[4] = {IF_SVE_GM_3A, IF_SVE_GN_3A, IF_SVE_GZ_3A, IF_SVE_HB_3A}; - const static insFormat formatEncode4I[4] = {IF_SVE_AX_1A, IF_SVE_AY_2A, IF_SVE_AZ_2A, IF_SVE_BA_3A}; - const static insFormat formatEncode4J[4] = {IF_SVE_BV_2A, IF_SVE_BV_2A_A, IF_SVE_CP_3A, IF_SVE_CQ_3A}; - const static insFormat formatEncode4K[4] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A, IF_SVE_IM_3A, IF_SVE_IN_4A}; - const static insFormat formatEncode4L[4] = {IF_SVE_IZ_4A, IF_SVE_IZ_4A_A, IF_SVE_JB_4A, IF_SVE_JM_3A}; - const static insFormat formatEncode3A[3] = {IF_SVE_AB_3A, IF_SVE_AT_3A, IF_SVE_EC_1A}; - const static insFormat formatEncode3B[3] = {IF_SVE_BH_3A, IF_SVE_BH_3B, IF_SVE_BH_3B_A}; - const static insFormat formatEncode3C[3] = {IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_EB_1A}; - const static insFormat formatEncode3D[3] = {IF_SVE_BR_3A, IF_SVE_BR_3B, IF_SVE_CI_3A}; - const static insFormat formatEncode3E[3] = {IF_SVE_AT_3A, IF_SVE_EC_1A, IF_SVE_ET_3A}; - const static insFormat formatEncode3F[3] = {IF_SVE_GU_3A, IF_SVE_GU_3B, IF_SVE_HU_4A}; - const static insFormat formatEncode3G[3] = {IF_SVE_GH_3A, IF_SVE_GH_3B, IF_SVE_GH_3B_B}; - const static insFormat formatEncode3H[3] = {IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; - const static insFormat formatEncode3I[3] = {IF_SVE_CM_3A, IF_SVE_CN_3A, IF_SVE_CO_3A}; - const static insFormat formatEncode3J[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3A}; - const static insFormat formatEncode3K[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3B}; - const static insFormat formatEncode3L[3] = {IF_SVE_DT_3A, IF_SVE_DX_3A, IF_SVE_DY_3A}; - const static insFormat formatEncode3M[3] = {IF_SVE_EJ_3A, IF_SVE_FA_3A, IF_SVE_FA_3B}; - const static insFormat formatEncode3N[3] = {IF_SVE_EK_3A, IF_SVE_FB_3A, IF_SVE_FB_3B}; - const static insFormat formatEncode3O[3] = {IF_SVE_EK_3A, IF_SVE_FC_3A, IF_SVE_FC_3B}; - const static insFormat formatEncode3P[3] = {IF_SVE_EL_3A, IF_SVE_FG_3A, IF_SVE_FG_3B}; - const static insFormat formatEncode3Q[3] = {IF_SVE_EO_3A, IF_SVE_FJ_3A, IF_SVE_FJ_3B}; - const static insFormat formatEncode3R[3] = {IF_SVE_FE_3A, IF_SVE_FE_3B, IF_SVE_FN_3A}; - const static insFormat formatEncode3S[3] = {IF_SVE_FH_3A, IF_SVE_FH_3B, IF_SVE_FN_3A}; - const static insFormat formatEncode3T[3] = {IF_SVE_GX_3C, IF_SVE_HK_3B, IF_SVE_HL_3B}; - const static insFormat formatEncode3U[3] = {IF_SVE_IM_3A, IF_SVE_IN_4A, IF_SVE_IX_4A}; - const static insFormat formatEncode3V[3] = {IF_SVE_JA_4A, IF_SVE_JB_4A, IF_SVE_JM_3A}; - const static insFormat formatEncode2AA[2] = {IF_SVE_ID_2A, IF_SVE_IE_2A}; - const static insFormat formatEncode2AB[2] = {IF_SVE_JG_2A, IF_SVE_JH_2A}; - const static insFormat formatEncode2AC[2] = {IF_SVE_AD_3A, IF_SVE_ED_1A}; - const static insFormat formatEncode2AD[2] = {IF_SVE_AB_3B, IF_SVE_AT_3B}; - const static insFormat formatEncode2AE[2] = {IF_SVE_CG_2A, IF_SVE_CJ_2A}; - const static insFormat formatEncode2AF[2] = {IF_SVE_AE_3A, IF_SVE_BD_3A}; - const static insFormat formatEncode2AG[2] = {IF_SVE_BS_1A, IF_SVE_CZ_4A}; - const static insFormat formatEncode2AH[2] = {IF_SVE_BQ_2A, IF_SVE_BQ_2B}; - const static insFormat formatEncode2AI[2] = {IF_SVE_AM_2A, IF_SVE_EU_3A}; - const static insFormat formatEncode2AJ[2] = {IF_SVE_HI_3A, IF_SVE_HT_4A}; - const static insFormat formatEncode2AK[2] = {IF_SVE_BZ_3A, IF_SVE_BZ_3A_A}; - const static insFormat formatEncode2AL[2] = {IF_SVE_GG_3A, IF_SVE_GG_3B}; - const static insFormat formatEncode2AM[2] = {IF_SVE_HL_3A, IF_SVE_HM_2A}; - const static insFormat formatEncode2AN[2] = {IF_SVE_EI_3A, IF_SVE_EZ_3A}; - const static insFormat formatEncode2AO[2] = {IF_SVE_GT_4A, IF_SVE_GV_3A}; - const static insFormat formatEncode2AP[2] = {IF_SVE_GY_3B, IF_SVE_HA_3A}; - const static insFormat formatEncode2AQ[2] = {IF_SVE_GO_3A, IF_SVE_HC_3A}; - const static insFormat formatEncode2AR[2] = {IF_SVE_AP_3A, IF_SVE_CZ_4A}; - const static insFormat formatEncode2AT[2] = {IF_SVE_AB_3A, IF_SVE_EC_1A}; - const static insFormat formatEncode2AU[2] = {IF_SVE_AH_3A, IF_SVE_BI_2A}; - const static insFormat formatEncode2AV[2] = {IF_SVE_BM_1A, IF_SVE_BN_1A}; - const static insFormat formatEncode2AW[2] = {IF_SVE_BO_1A, IF_SVE_BP_1A}; - const static insFormat formatEncode2AX[2] = {IF_SVE_CC_2A, IF_SVE_CD_2A}; - const static insFormat formatEncode2AY[2] = {IF_SVE_CR_3A, IF_SVE_CS_3A}; - const static insFormat formatEncode2AZ[2] = {IF_SVE_CV_3A, IF_SVE_CV_3B}; - const static insFormat formatEncode2BA[2] = {IF_SVE_CW_4A, IF_SVE_CZ_4A}; - const static insFormat formatEncode2BB[2] = {IF_SVE_CZ_4A, IF_SVE_CZ_4A_A}; - const static insFormat formatEncode2BC[2] = {IF_SVE_DE_1A, IF_SVE_DZ_1A}; - const static insFormat formatEncode2BD[2] = {IF_SVE_DG_2A, IF_SVE_DH_1A}; - const static insFormat formatEncode2BE[2] = {IF_SVE_DK_3A, IF_SVE_DL_2A}; - const static insFormat formatEncode2BF[2] = {IF_SVE_DM_2A, IF_SVE_DN_2A}; - const static insFormat formatEncode2BG[2] = {IF_SVE_DO_2A, IF_SVE_DP_2A}; - const static insFormat formatEncode2BH[2] = {IF_SVE_DW_2A, IF_SVE_DW_2B}; - const static insFormat formatEncode2BI[2] = {IF_SVE_FN_3A, IF_SVE_FN_3B}; - const static insFormat formatEncode2BJ[2] = {IF_SVE_GQ_3A, IF_SVE_HG_2A}; - const static insFormat formatEncode2BK[2] = {IF_SVE_GU_3C, IF_SVE_HU_4B}; - const static insFormat formatEncode2BL[2] = {IF_SVE_GZ_3A, IF_SVE_HB_3A}; - const static insFormat formatEncode2BM[2] = {IF_SVE_HK_3B, IF_SVE_HL_3B}; - const static insFormat formatEncode2BN[2] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A}; - const static insFormat formatEncode2BO[2] = {IF_SVE_IO_3A, IF_SVE_IP_4A}; - const static insFormat formatEncode2BP[2] = {IF_SVE_IQ_3A, IF_SVE_IR_4A}; - const static insFormat formatEncode2BQ[2] = {IF_SVE_IS_3A, IF_SVE_IT_4A}; - const static insFormat formatEncode2BR[2] = {IF_SVE_JC_4A, IF_SVE_JO_3A}; - const static insFormat formatEncode2BS[2] = {IF_SVE_JE_3A, IF_SVE_JF_4A}; - - code_t code = BAD_CODE; - insFormat insFmt = emitInsFormat(ins); - bool encoding_found = false; - int index = -1; - - switch (insFmt) - { - case IF_SVE_13A: - for (index = 0; index < 13; index++) - { - if (fmt == formatEncode13A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_11A: - for (index = 0; index < 11; index++) - { - if (fmt == formatEncode11A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9A: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9B: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9C: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9D: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9E: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_9F: - for (index = 0; index < 9; index++) - { - if (fmt == formatEncode9F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_8A: - for (index = 0; index < 8; index++) - { - if (fmt == formatEncode8A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_8B: - for (index = 0; index < 8; index++) - { - if (fmt == formatEncode8B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_8C: - for (index = 0; index < 8; index++) - { - if (fmt == formatEncode8C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_7A: - for (index = 0; index < 7; index++) - { - if (fmt == formatEncode7A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6A: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6B: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6C: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6D: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6E: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6F: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_6G: - for (index = 0; index < 6; index++) - { - if (fmt == formatEncode6G[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5A: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5B: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5C: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5D: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_5E: - for (index = 0; index < 5; index++) - { - if (fmt == formatEncode5E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4A: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4B: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4E: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4F: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4G: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4G[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4H: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4H[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4I: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4I[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4J: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4J[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4K: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4K[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_4L: - for (index = 0; index < 4; index++) - { - if (fmt == formatEncode4L[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3A: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3A[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3B: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3B[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3C: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3C[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3D: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3D[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3E: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3E[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3F: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3F[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3G: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3G[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3H: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3H[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3I: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3I[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3J: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3J[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3K: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3K[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3L: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3L[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3M: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3M[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3N: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3N[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3O: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3O[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3P: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3P[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3Q: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3Q[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3R: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3R[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3S: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3S[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3T: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3T[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3U: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3U[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_3V: - for (index = 0; index < 3; index++) - { - if (fmt == formatEncode3V[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AA: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AA[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AB: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AB[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AC: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AC[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AD: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AD[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AE: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AE[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AF: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AF[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AG: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AG[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AH: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AH[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AI: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AI[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AJ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AJ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AK: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AK[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AL: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AL[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AM: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AM[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AN: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AN[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AO: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AO[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AP: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AP[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AQ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AQ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AR: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AR[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AT: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AT[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AU: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AU[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AV: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AV[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AW: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AW[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AX: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AX[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AY: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AY[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2AZ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2AZ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BA: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BA[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BB: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BB[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BC: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BC[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BD: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BD[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BE: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BE[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BF: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BF[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BG: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BG[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BH: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BH[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BI: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BI[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BJ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BJ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BK: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BK[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BL: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BL[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BM: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BM[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BN: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BN[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BO: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BO[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BP: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BP[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BQ: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BQ[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BR: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BR[index]) - { - encoding_found = true; - break; - } - } - break; - case IF_SVE_2BS: - for (index = 0; index < 2; index++) - { - if (fmt == formatEncode2BS[index]) - { - encoding_found = true; - break; - } - } - break; - default: - if (fmt == insFmt) - { - encoding_found = true; - index = 0; - } - else - { - encoding_found = false; - } - break; - } - - assert(encoding_found); - const unsigned sve_ins_offset = ((unsigned)ins - INS_sve_invalid); - - switch (index) - { - case 0: - assert(sve_ins_offset < ArrLen(insCodes1)); - code = insCodes1[sve_ins_offset]; - break; - case 1: - assert(sve_ins_offset < ArrLen(insCodes2)); - code = insCodes2[sve_ins_offset]; - break; - case 2: - assert(sve_ins_offset < ArrLen(insCodes3)); - code = insCodes3[sve_ins_offset]; - break; - case 3: - assert(sve_ins_offset < ArrLen(insCodes4)); - code = insCodes4[sve_ins_offset]; - break; - case 4: - assert(sve_ins_offset < ArrLen(insCodes5)); - code = insCodes5[sve_ins_offset]; - break; - case 5: - assert(sve_ins_offset < ArrLen(insCodes6)); - code = insCodes6[sve_ins_offset]; - break; - case 6: - assert(sve_ins_offset < ArrLen(insCodes7)); - code = insCodes7[sve_ins_offset]; - break; - case 7: - assert(sve_ins_offset < ArrLen(insCodes8)); - code = insCodes8[sve_ins_offset]; - break; - case 8: - assert(sve_ins_offset < ArrLen(insCodes9)); - code = insCodes9[sve_ins_offset]; - break; - case 9: - assert(sve_ins_offset < ArrLen(insCodes10)); - code = insCodes10[sve_ins_offset]; - break; - case 10: - assert(sve_ins_offset < ArrLen(insCodes11)); - code = insCodes11[sve_ins_offset]; - break; - case 11: - assert(sve_ins_offset < ArrLen(insCodes12)); - code = insCodes12[sve_ins_offset]; - break; - case 12: - assert(sve_ins_offset < ArrLen(insCodes13)); - code = insCodes13[sve_ins_offset]; - break; - } - - assert((code != BAD_CODE)); - - return code; -} - // true if this 'imm' can be encoded as a input operand to a mov instruction /*static*/ bool emitter::emitIns_valid_imm_for_mov(INT64 imm, emitAttr size) { diff --git a/src/coreclr/jit/emitarm64sve.cpp b/src/coreclr/jit/emitarm64sve.cpp index 99d7d3f9a78be..577d815379dbe 100644 --- a/src/coreclr/jit/emitarm64sve.cpp +++ b/src/coreclr/jit/emitarm64sve.cpp @@ -66,6 +66,1438 @@ static const char * const svePatternNames[] = // clang-format on +/***************************************************************************** + * + * Returns the specific encoding of the given CPU instruction and format + */ + +emitter::code_t emitter::emitInsCodeSve(instruction ins, insFormat fmt) +{ + // clang-format off + const static code_t insCodes1[] = + { + #define INST1(id, nm, info, fmt, e1 ) e1, + #define INST2(id, nm, info, fmt, e1, e2 ) e1, + #define INST3(id, nm, info, fmt, e1, e2, e3 ) e1, + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e1, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e1, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e1, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e1, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e1, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e1, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e1, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e1, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes2[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) e2, + #define INST3(id, nm, info, fmt, e1, e2, e3 ) e2, + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e2, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e2, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e2, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e2, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e2, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e2, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e2, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e2, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes3[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) e3, + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e3, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e3, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e3, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e3, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e3, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e3, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e3, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e3, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes4[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) e4, + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e4, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e4, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e4, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e4, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e4, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e4, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e4, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes5[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) e5, + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e5, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e5, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e5, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e5, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e5, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e5, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes6[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) e6, + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e6, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e6, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e6, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e6, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e6, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes7[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) e7, + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e7, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e7, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e7, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e7, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes8[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) e8, + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e8, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e8, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e8, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes9[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) e9, + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e9, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e9, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes10[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e10, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e10, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes11[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) e11, + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e11, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes12[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e12, + #include "instrsarm64sve.h" + }; + + const static code_t insCodes13[] = + { + #define INST1(id, nm, info, fmt, e1 ) + #define INST2(id, nm, info, fmt, e1, e2 ) + #define INST3(id, nm, info, fmt, e1, e2, e3 ) + #define INST4(id, nm, info, fmt, e1, e2, e3, e4 ) + #define INST5(id, nm, info, fmt, e1, e2, e3, e4, e5 ) + #define INST6(id, nm, info, fmt, e1, e2, e3, e4, e5, e6 ) + #define INST7(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7 ) + #define INST8(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8 ) + #define INST9(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9 ) + #define INST11(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11 ) + #define INST13(id, nm, info, fmt, e1, e2, e3, e4, e5, e6, e7, e8, e9, e10, e11, e12, e13 ) e13, + #include "instrsarm64sve.h" + }; + + // clang-format on + const static insFormat formatEncode13A[13] = {IF_SVE_AU_3A, IF_SVE_BT_1A, IF_SVE_BV_2A, IF_SVE_BV_2A_J, + IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_CP_3A, IF_SVE_CQ_3A, + IF_SVE_CW_4A, IF_SVE_CZ_4A, IF_SVE_CZ_4A_K, IF_SVE_CZ_4A_L, + IF_SVE_EB_1A}; + const static insFormat formatEncode11A[11] = {IF_SVE_JD_4B, IF_SVE_JD_4C, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, + IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, IF_SVE_JJ_4B, + IF_SVE_JJ_4B_E, IF_SVE_JN_3B, IF_SVE_JN_3C}; + const static insFormat formatEncode9A[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, + IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, + IF_SVE_HX_3A_E, IF_SVE_IJ_3A_F, IF_SVE_IK_4A_G}; + const static insFormat formatEncode9B[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, + IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, + IF_SVE_HX_3A_E, IF_SVE_IJ_3A_G, IF_SVE_IK_4A_I}; + const static insFormat formatEncode9C[9] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, + IF_SVE_HW_4A_C, IF_SVE_HW_4B, IF_SVE_HW_4B_D, + IF_SVE_HX_3A_E, IF_SVE_IH_3A_F, IF_SVE_II_4A_H}; + const static insFormat formatEncode9D[9] = {IF_SVE_IH_3A, IF_SVE_IH_3A_A, IF_SVE_II_4A, + IF_SVE_II_4A_B, IF_SVE_IU_4A, IF_SVE_IU_4A_C, + IF_SVE_IU_4B, IF_SVE_IU_4B_D, IF_SVE_IV_3A}; + const static insFormat formatEncode9E[9] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JJ_4A, + IF_SVE_JJ_4A_B, IF_SVE_JJ_4A_C, IF_SVE_JJ_4A_D, + IF_SVE_JJ_4B, IF_SVE_JJ_4B_E, IF_SVE_JN_3A}; + const static insFormat formatEncode9F[9] = {IF_SVE_JD_4C, IF_SVE_JD_4C_A, IF_SVE_JJ_4A, + IF_SVE_JJ_4A_B, IF_SVE_JJ_4B, IF_SVE_JJ_4B_C, + IF_SVE_JL_3A, IF_SVE_JN_3C, IF_SVE_JN_3C_D}; + const static insFormat formatEncode8A[8] = {IF_SVE_CE_2A, IF_SVE_CE_2B, IF_SVE_CE_2C, IF_SVE_CE_2D, + IF_SVE_CF_2A, IF_SVE_CF_2B, IF_SVE_CF_2C, IF_SVE_CF_2D}; + const static insFormat formatEncode8B[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, + IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_F}; + const static insFormat formatEncode8C[8] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4A_B, IF_SVE_HW_4A_C, + IF_SVE_HW_4B, IF_SVE_HW_4B_D, IF_SVE_HX_3A_E, IF_SVE_IG_4A_G}; + const static insFormat formatEncode7A[7] = {IF_SVE_IJ_3A, IF_SVE_IK_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, + IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; + const static insFormat formatEncode6A[6] = {IF_SVE_AE_3A, IF_SVE_BD_3A, IF_SVE_EE_1A, + IF_SVE_FD_3A, IF_SVE_FD_3B, IF_SVE_FD_3C}; + const static insFormat formatEncode6B[6] = {IF_SVE_GY_3A, IF_SVE_GY_3B, IF_SVE_GY_3B_D, + IF_SVE_HA_3A, IF_SVE_HA_3A_E, IF_SVE_HA_3A_F}; + const static insFormat formatEncode6C[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, + IF_SVE_HX_3A_B, IF_SVE_IJ_3A_D, IF_SVE_IK_4A_F}; + const static insFormat formatEncode6D[6] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, + IF_SVE_HX_3A_B, IF_SVE_IJ_3A_E, IF_SVE_IK_4A_H}; + const static insFormat formatEncode6E[6] = {IF_SVE_HY_3A, IF_SVE_HY_3A_A, IF_SVE_HY_3B, + IF_SVE_HZ_2A_B, IF_SVE_IA_2A, IF_SVE_IB_3A}; + const static insFormat formatEncode6F[6] = {IF_SVE_IG_4A, IF_SVE_IU_4A, IF_SVE_IU_4A_A, + IF_SVE_IU_4B, IF_SVE_IU_4B_B, IF_SVE_IV_3A}; + const static insFormat formatEncode6G[6] = {IF_SVE_JD_4A, IF_SVE_JI_3A_A, IF_SVE_JK_4A, + IF_SVE_JK_4A_B, IF_SVE_JK_4B, IF_SVE_JN_3A}; + const static insFormat formatEncode5A[5] = {IF_SVE_AM_2A, IF_SVE_AN_3A, IF_SVE_AO_3A, IF_SVE_BF_2A, IF_SVE_BG_3A}; + const static insFormat formatEncode5B[5] = {IF_SVE_GX_3A, IF_SVE_GX_3B, IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; + const static insFormat formatEncode5C[5] = {IF_SVE_EF_3A, IF_SVE_EG_3A, IF_SVE_EH_3A, IF_SVE_EY_3A, IF_SVE_EY_3B}; + const static insFormat formatEncode5D[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, + IF_SVE_IG_4A_D}; + const static insFormat formatEncode5E[5] = {IF_SVE_HW_4A, IF_SVE_HW_4A_A, IF_SVE_HW_4B, IF_SVE_HX_3A_B, + IF_SVE_IG_4A_E}; + const static insFormat formatEncode4A[4] = {IF_SVE_AA_3A, IF_SVE_AU_3A, IF_SVE_BS_1A, IF_SVE_CZ_4A}; + const static insFormat formatEncode4B[4] = {IF_SVE_BU_2A, IF_SVE_BV_2B, IF_SVE_EA_1A, IF_SVE_EB_1B}; + const static insFormat formatEncode4E[4] = {IF_SVE_BE_3A, IF_SVE_FI_3A, IF_SVE_FI_3B, IF_SVE_FI_3C}; + const static insFormat formatEncode4F[4] = {IF_SVE_EM_3A, IF_SVE_FK_3A, IF_SVE_FK_3B, IF_SVE_FK_3C}; + const static insFormat formatEncode4G[4] = {IF_SVE_AR_4A, IF_SVE_FF_3A, IF_SVE_FF_3B, IF_SVE_FF_3C}; + const static insFormat formatEncode4H[4] = {IF_SVE_GM_3A, IF_SVE_GN_3A, IF_SVE_GZ_3A, IF_SVE_HB_3A}; + const static insFormat formatEncode4I[4] = {IF_SVE_AX_1A, IF_SVE_AY_2A, IF_SVE_AZ_2A, IF_SVE_BA_3A}; + const static insFormat formatEncode4J[4] = {IF_SVE_BV_2A, IF_SVE_BV_2A_A, IF_SVE_CP_3A, IF_SVE_CQ_3A}; + const static insFormat formatEncode4K[4] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A, IF_SVE_IM_3A, IF_SVE_IN_4A}; + const static insFormat formatEncode4L[4] = {IF_SVE_IZ_4A, IF_SVE_IZ_4A_A, IF_SVE_JB_4A, IF_SVE_JM_3A}; + const static insFormat formatEncode3A[3] = {IF_SVE_AB_3A, IF_SVE_AT_3A, IF_SVE_EC_1A}; + const static insFormat formatEncode3B[3] = {IF_SVE_BH_3A, IF_SVE_BH_3B, IF_SVE_BH_3B_A}; + const static insFormat formatEncode3C[3] = {IF_SVE_BW_2A, IF_SVE_CB_2A, IF_SVE_EB_1A}; + const static insFormat formatEncode3D[3] = {IF_SVE_BR_3A, IF_SVE_BR_3B, IF_SVE_CI_3A}; + const static insFormat formatEncode3E[3] = {IF_SVE_AT_3A, IF_SVE_EC_1A, IF_SVE_ET_3A}; + const static insFormat formatEncode3F[3] = {IF_SVE_GU_3A, IF_SVE_GU_3B, IF_SVE_HU_4A}; + const static insFormat formatEncode3G[3] = {IF_SVE_GH_3A, IF_SVE_GH_3B, IF_SVE_GH_3B_B}; + const static insFormat formatEncode3H[3] = {IF_SVE_HK_3A, IF_SVE_HL_3A, IF_SVE_HM_2A}; + const static insFormat formatEncode3I[3] = {IF_SVE_CM_3A, IF_SVE_CN_3A, IF_SVE_CO_3A}; + const static insFormat formatEncode3J[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3A}; + const static insFormat formatEncode3K[3] = {IF_SVE_CX_4A, IF_SVE_CX_4A_A, IF_SVE_CY_3B}; + const static insFormat formatEncode3L[3] = {IF_SVE_DT_3A, IF_SVE_DX_3A, IF_SVE_DY_3A}; + const static insFormat formatEncode3M[3] = {IF_SVE_EJ_3A, IF_SVE_FA_3A, IF_SVE_FA_3B}; + const static insFormat formatEncode3N[3] = {IF_SVE_EK_3A, IF_SVE_FB_3A, IF_SVE_FB_3B}; + const static insFormat formatEncode3O[3] = {IF_SVE_EK_3A, IF_SVE_FC_3A, IF_SVE_FC_3B}; + const static insFormat formatEncode3P[3] = {IF_SVE_EL_3A, IF_SVE_FG_3A, IF_SVE_FG_3B}; + const static insFormat formatEncode3Q[3] = {IF_SVE_EO_3A, IF_SVE_FJ_3A, IF_SVE_FJ_3B}; + const static insFormat formatEncode3R[3] = {IF_SVE_FE_3A, IF_SVE_FE_3B, IF_SVE_FN_3A}; + const static insFormat formatEncode3S[3] = {IF_SVE_FH_3A, IF_SVE_FH_3B, IF_SVE_FN_3A}; + const static insFormat formatEncode3T[3] = {IF_SVE_GX_3C, IF_SVE_HK_3B, IF_SVE_HL_3B}; + const static insFormat formatEncode3U[3] = {IF_SVE_IM_3A, IF_SVE_IN_4A, IF_SVE_IX_4A}; + const static insFormat formatEncode3V[3] = {IF_SVE_JA_4A, IF_SVE_JB_4A, IF_SVE_JM_3A}; + const static insFormat formatEncode2AA[2] = {IF_SVE_ID_2A, IF_SVE_IE_2A}; + const static insFormat formatEncode2AB[2] = {IF_SVE_JG_2A, IF_SVE_JH_2A}; + const static insFormat formatEncode2AC[2] = {IF_SVE_AD_3A, IF_SVE_ED_1A}; + const static insFormat formatEncode2AD[2] = {IF_SVE_AB_3B, IF_SVE_AT_3B}; + const static insFormat formatEncode2AE[2] = {IF_SVE_CG_2A, IF_SVE_CJ_2A}; + const static insFormat formatEncode2AF[2] = {IF_SVE_AE_3A, IF_SVE_BD_3A}; + const static insFormat formatEncode2AG[2] = {IF_SVE_BS_1A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2AH[2] = {IF_SVE_BQ_2A, IF_SVE_BQ_2B}; + const static insFormat formatEncode2AI[2] = {IF_SVE_AM_2A, IF_SVE_EU_3A}; + const static insFormat formatEncode2AJ[2] = {IF_SVE_HI_3A, IF_SVE_HT_4A}; + const static insFormat formatEncode2AK[2] = {IF_SVE_BZ_3A, IF_SVE_BZ_3A_A}; + const static insFormat formatEncode2AL[2] = {IF_SVE_GG_3A, IF_SVE_GG_3B}; + const static insFormat formatEncode2AM[2] = {IF_SVE_HL_3A, IF_SVE_HM_2A}; + const static insFormat formatEncode2AN[2] = {IF_SVE_EI_3A, IF_SVE_EZ_3A}; + const static insFormat formatEncode2AO[2] = {IF_SVE_GT_4A, IF_SVE_GV_3A}; + const static insFormat formatEncode2AP[2] = {IF_SVE_GY_3B, IF_SVE_HA_3A}; + const static insFormat formatEncode2AQ[2] = {IF_SVE_GO_3A, IF_SVE_HC_3A}; + const static insFormat formatEncode2AR[2] = {IF_SVE_AP_3A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2AT[2] = {IF_SVE_AB_3A, IF_SVE_EC_1A}; + const static insFormat formatEncode2AU[2] = {IF_SVE_AH_3A, IF_SVE_BI_2A}; + const static insFormat formatEncode2AV[2] = {IF_SVE_BM_1A, IF_SVE_BN_1A}; + const static insFormat formatEncode2AW[2] = {IF_SVE_BO_1A, IF_SVE_BP_1A}; + const static insFormat formatEncode2AX[2] = {IF_SVE_CC_2A, IF_SVE_CD_2A}; + const static insFormat formatEncode2AY[2] = {IF_SVE_CR_3A, IF_SVE_CS_3A}; + const static insFormat formatEncode2AZ[2] = {IF_SVE_CV_3A, IF_SVE_CV_3B}; + const static insFormat formatEncode2BA[2] = {IF_SVE_CW_4A, IF_SVE_CZ_4A}; + const static insFormat formatEncode2BB[2] = {IF_SVE_CZ_4A, IF_SVE_CZ_4A_A}; + const static insFormat formatEncode2BC[2] = {IF_SVE_DE_1A, IF_SVE_DZ_1A}; + const static insFormat formatEncode2BD[2] = {IF_SVE_DG_2A, IF_SVE_DH_1A}; + const static insFormat formatEncode2BE[2] = {IF_SVE_DK_3A, IF_SVE_DL_2A}; + const static insFormat formatEncode2BF[2] = {IF_SVE_DM_2A, IF_SVE_DN_2A}; + const static insFormat formatEncode2BG[2] = {IF_SVE_DO_2A, IF_SVE_DP_2A}; + const static insFormat formatEncode2BH[2] = {IF_SVE_DW_2A, IF_SVE_DW_2B}; + const static insFormat formatEncode2BI[2] = {IF_SVE_FN_3A, IF_SVE_FN_3B}; + const static insFormat formatEncode2BJ[2] = {IF_SVE_GQ_3A, IF_SVE_HG_2A}; + const static insFormat formatEncode2BK[2] = {IF_SVE_GU_3C, IF_SVE_HU_4B}; + const static insFormat formatEncode2BL[2] = {IF_SVE_GZ_3A, IF_SVE_HB_3A}; + const static insFormat formatEncode2BM[2] = {IF_SVE_HK_3B, IF_SVE_HL_3B}; + const static insFormat formatEncode2BN[2] = {IF_SVE_IF_4A, IF_SVE_IF_4A_A}; + const static insFormat formatEncode2BO[2] = {IF_SVE_IO_3A, IF_SVE_IP_4A}; + const static insFormat formatEncode2BP[2] = {IF_SVE_IQ_3A, IF_SVE_IR_4A}; + const static insFormat formatEncode2BQ[2] = {IF_SVE_IS_3A, IF_SVE_IT_4A}; + const static insFormat formatEncode2BR[2] = {IF_SVE_JC_4A, IF_SVE_JO_3A}; + const static insFormat formatEncode2BS[2] = {IF_SVE_JE_3A, IF_SVE_JF_4A}; + + code_t code = BAD_CODE; + insFormat insFmt = emitInsFormat(ins); + bool encoding_found = false; + int index = -1; + + switch (insFmt) + { + case IF_SVE_13A: + for (index = 0; index < 13; index++) + { + if (fmt == formatEncode13A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_11A: + for (index = 0; index < 11; index++) + { + if (fmt == formatEncode11A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_9A: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_9B: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9B[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_9C: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9C[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_9D: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9D[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_9E: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9E[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_9F: + for (index = 0; index < 9; index++) + { + if (fmt == formatEncode9F[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_8A: + for (index = 0; index < 8; index++) + { + if (fmt == formatEncode8A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_8B: + for (index = 0; index < 8; index++) + { + if (fmt == formatEncode8B[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_8C: + for (index = 0; index < 8; index++) + { + if (fmt == formatEncode8C[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_7A: + for (index = 0; index < 7; index++) + { + if (fmt == formatEncode7A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6A: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6B: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6B[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6C: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6C[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6D: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6D[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6E: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6E[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6F: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6F[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_6G: + for (index = 0; index < 6; index++) + { + if (fmt == formatEncode6G[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_5A: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_5B: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5B[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_5C: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5C[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_5D: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5D[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_5E: + for (index = 0; index < 5; index++) + { + if (fmt == formatEncode5E[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4A: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4B: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4B[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4E: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4E[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4F: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4F[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4G: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4G[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4H: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4H[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4I: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4I[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4J: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4J[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4K: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4K[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_4L: + for (index = 0; index < 4; index++) + { + if (fmt == formatEncode4L[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3A: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3A[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3B: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3B[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3C: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3C[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3D: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3D[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3E: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3E[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3F: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3F[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3G: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3G[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3H: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3H[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3I: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3I[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3J: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3J[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3K: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3K[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3L: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3L[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3M: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3M[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3N: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3N[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3O: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3O[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3P: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3P[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3Q: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3Q[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3R: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3R[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3S: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3S[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3T: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3T[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3U: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3U[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_3V: + for (index = 0; index < 3; index++) + { + if (fmt == formatEncode3V[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AA: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AA[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AB: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AB[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AC: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AC[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AD: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AD[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AE: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AE[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AF: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AF[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AG: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AG[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AH: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AH[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AI: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AI[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AJ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AJ[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AK: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AK[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AL: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AL[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AM: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AM[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AN: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AN[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AO: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AO[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AP: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AP[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AQ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AQ[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AR: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AR[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AT: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AT[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AU: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AU[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AV: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AV[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AW: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AW[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AX: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AX[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AY: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AY[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2AZ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2AZ[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BA: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BA[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BB: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BB[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BC: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BC[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BD: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BD[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BE: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BE[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BF: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BF[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BG: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BG[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BH: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BH[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BI: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BI[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BJ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BJ[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BK: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BK[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BL: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BL[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BM: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BM[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BN: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BN[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BO: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BO[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BP: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BP[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BQ: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BQ[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BR: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BR[index]) + { + encoding_found = true; + break; + } + } + break; + case IF_SVE_2BS: + for (index = 0; index < 2; index++) + { + if (fmt == formatEncode2BS[index]) + { + encoding_found = true; + break; + } + } + break; + default: + if (fmt == insFmt) + { + encoding_found = true; + index = 0; + } + else + { + encoding_found = false; + } + break; + } + + assert(encoding_found); + const unsigned sve_ins_offset = ((unsigned)ins - INS_sve_invalid); + + switch (index) + { + case 0: + assert(sve_ins_offset < ArrLen(insCodes1)); + code = insCodes1[sve_ins_offset]; + break; + case 1: + assert(sve_ins_offset < ArrLen(insCodes2)); + code = insCodes2[sve_ins_offset]; + break; + case 2: + assert(sve_ins_offset < ArrLen(insCodes3)); + code = insCodes3[sve_ins_offset]; + break; + case 3: + assert(sve_ins_offset < ArrLen(insCodes4)); + code = insCodes4[sve_ins_offset]; + break; + case 4: + assert(sve_ins_offset < ArrLen(insCodes5)); + code = insCodes5[sve_ins_offset]; + break; + case 5: + assert(sve_ins_offset < ArrLen(insCodes6)); + code = insCodes6[sve_ins_offset]; + break; + case 6: + assert(sve_ins_offset < ArrLen(insCodes7)); + code = insCodes7[sve_ins_offset]; + break; + case 7: + assert(sve_ins_offset < ArrLen(insCodes8)); + code = insCodes8[sve_ins_offset]; + break; + case 8: + assert(sve_ins_offset < ArrLen(insCodes9)); + code = insCodes9[sve_ins_offset]; + break; + case 9: + assert(sve_ins_offset < ArrLen(insCodes10)); + code = insCodes10[sve_ins_offset]; + break; + case 10: + assert(sve_ins_offset < ArrLen(insCodes11)); + code = insCodes11[sve_ins_offset]; + break; + case 11: + assert(sve_ins_offset < ArrLen(insCodes12)); + code = insCodes12[sve_ins_offset]; + break; + case 12: + assert(sve_ins_offset < ArrLen(insCodes13)); + code = insCodes13[sve_ins_offset]; + break; + } + + assert((code != BAD_CODE)); + + return code; +} + /***************************************************************************** * * Add a SVE instruction with a single immediate value.